2024-11-28 06:53:39,671 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-28 06:53:39,689 main DEBUG Took 0.015121 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-28 06:53:39,690 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-28 06:53:39,690 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-28 06:53:39,692 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-28 06:53:39,693 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 06:53:39,704 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-28 06:53:39,722 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 06:53:39,724 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 06:53:39,725 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 06:53:39,726 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 06:53:39,727 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 06:53:39,727 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 06:53:39,729 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 06:53:39,729 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 06:53:39,730 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 06:53:39,731 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 06:53:39,732 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 06:53:39,733 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 06:53:39,734 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 06:53:39,734 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 06:53:39,735 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 06:53:39,735 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 06:53:39,736 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 06:53:39,737 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 06:53:39,737 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 06:53:39,738 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 06:53:39,739 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 06:53:39,739 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 06:53:39,740 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 06:53:39,741 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 06:53:39,741 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 06:53:39,742 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-28 06:53:39,745 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 06:53:39,747 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-28 06:53:39,750 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-28 06:53:39,751 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-28 06:53:39,753 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-28 06:53:39,754 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-28 06:53:39,765 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-28 06:53:39,769 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-28 06:53:39,772 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-28 06:53:39,772 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-28 06:53:39,773 main DEBUG createAppenders(={Console}) 2024-11-28 06:53:39,774 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-28 06:53:39,774 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-28 06:53:39,775 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-28 06:53:39,776 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-28 06:53:39,776 main DEBUG OutputStream closed 2024-11-28 06:53:39,777 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-28 06:53:39,777 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-28 06:53:39,778 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-28 06:53:39,878 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-28 06:53:39,881 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-28 06:53:39,882 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-28 06:53:39,884 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-28 06:53:39,884 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-28 06:53:39,885 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-28 06:53:39,886 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-28 06:53:39,886 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-28 06:53:39,887 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-28 06:53:39,887 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-28 06:53:39,888 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-28 06:53:39,888 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-28 06:53:39,889 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-28 06:53:39,889 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-28 06:53:39,890 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-28 06:53:39,890 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-28 06:53:39,891 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-28 06:53:39,892 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-28 06:53:39,894 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-28 06:53:39,894 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-28 06:53:39,895 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-28 06:53:39,895 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-28T06:53:40,148 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ce3257b-8be6-c2e3-213a-03473f81f2f4 2024-11-28 06:53:40,150 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-28 06:53:40,151 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-28T06:53:40,159 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-28T06:53:40,180 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-28T06:53:40,203 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ce3257b-8be6-c2e3-213a-03473f81f2f4/cluster_b8990226-b232-1028-e659-af5834b9be30, deleteOnExit=true 2024-11-28T06:53:40,203 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-28T06:53:40,205 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ce3257b-8be6-c2e3-213a-03473f81f2f4/test.cache.data in system properties and HBase conf 2024-11-28T06:53:40,206 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ce3257b-8be6-c2e3-213a-03473f81f2f4/hadoop.tmp.dir in system properties and HBase conf 2024-11-28T06:53:40,207 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ce3257b-8be6-c2e3-213a-03473f81f2f4/hadoop.log.dir in system properties and HBase conf 2024-11-28T06:53:40,208 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ce3257b-8be6-c2e3-213a-03473f81f2f4/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-28T06:53:40,209 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ce3257b-8be6-c2e3-213a-03473f81f2f4/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-28T06:53:40,210 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-28T06:53:40,328 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-28T06:53:40,435 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T06:53:40,439 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ce3257b-8be6-c2e3-213a-03473f81f2f4/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-28T06:53:40,440 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ce3257b-8be6-c2e3-213a-03473f81f2f4/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-28T06:53:40,440 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ce3257b-8be6-c2e3-213a-03473f81f2f4/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-28T06:53:40,441 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ce3257b-8be6-c2e3-213a-03473f81f2f4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-28T06:53:40,442 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ce3257b-8be6-c2e3-213a-03473f81f2f4/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-28T06:53:40,442 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ce3257b-8be6-c2e3-213a-03473f81f2f4/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-28T06:53:40,443 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ce3257b-8be6-c2e3-213a-03473f81f2f4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-28T06:53:40,443 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ce3257b-8be6-c2e3-213a-03473f81f2f4/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-28T06:53:40,444 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ce3257b-8be6-c2e3-213a-03473f81f2f4/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-28T06:53:40,444 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ce3257b-8be6-c2e3-213a-03473f81f2f4/nfs.dump.dir in system properties and HBase conf 2024-11-28T06:53:40,445 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ce3257b-8be6-c2e3-213a-03473f81f2f4/java.io.tmpdir in system properties and HBase conf 2024-11-28T06:53:40,445 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ce3257b-8be6-c2e3-213a-03473f81f2f4/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-28T06:53:40,446 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ce3257b-8be6-c2e3-213a-03473f81f2f4/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-28T06:53:40,446 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ce3257b-8be6-c2e3-213a-03473f81f2f4/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-28T06:53:41,690 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-28T06:53:41,773 INFO [Time-limited test {}] log.Log(170): Logging initialized @2927ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-28T06:53:41,847 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T06:53:41,911 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-28T06:53:41,935 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-28T06:53:41,937 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-28T06:53:41,938 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-28T06:53:41,951 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T06:53:41,954 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ce3257b-8be6-c2e3-213a-03473f81f2f4/hadoop.log.dir/,AVAILABLE} 2024-11-28T06:53:41,956 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-28T06:53:42,142 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ce3257b-8be6-c2e3-213a-03473f81f2f4/java.io.tmpdir/jetty-localhost-45835-hadoop-hdfs-3_4_1-tests_jar-_-any-18266341381020285107/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-28T06:53:42,156 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:45835} 2024-11-28T06:53:42,157 INFO [Time-limited test {}] server.Server(415): Started @3311ms 2024-11-28T06:53:42,664 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T06:53:42,672 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-28T06:53:42,673 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-28T06:53:42,674 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-28T06:53:42,674 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-28T06:53:42,675 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ce3257b-8be6-c2e3-213a-03473f81f2f4/hadoop.log.dir/,AVAILABLE} 2024-11-28T06:53:42,675 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-28T06:53:42,797 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ce3257b-8be6-c2e3-213a-03473f81f2f4/java.io.tmpdir/jetty-localhost-42687-hadoop-hdfs-3_4_1-tests_jar-_-any-14915192396482440429/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-28T06:53:42,798 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:42687} 2024-11-28T06:53:42,798 INFO [Time-limited test {}] server.Server(415): Started @3952ms 2024-11-28T06:53:42,845 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-28T06:53:43,734 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ce3257b-8be6-c2e3-213a-03473f81f2f4/cluster_b8990226-b232-1028-e659-af5834b9be30/data/data2/current/BP-830994580-172.17.0.2-1732776821123/current, will proceed with Du for space computation calculation, 2024-11-28T06:53:43,734 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ce3257b-8be6-c2e3-213a-03473f81f2f4/cluster_b8990226-b232-1028-e659-af5834b9be30/data/data1/current/BP-830994580-172.17.0.2-1732776821123/current, will proceed with Du for space computation calculation, 2024-11-28T06:53:43,766 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-28T06:53:43,811 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x65b9b11602ead7e3 with lease ID 0x50e443b2445aac48: Processing first storage report for DS-668c083b-bcb6-429c-8335-603571730fb1 from datanode DatanodeRegistration(127.0.0.1:33259, datanodeUuid=76be8b2a-67e3-435a-b5f7-adea186d1f20, infoPort=35803, infoSecurePort=0, ipcPort=35437, storageInfo=lv=-57;cid=testClusterID;nsid=1702142786;c=1732776821123) 2024-11-28T06:53:43,813 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x65b9b11602ead7e3 with lease ID 0x50e443b2445aac48: from storage DS-668c083b-bcb6-429c-8335-603571730fb1 node DatanodeRegistration(127.0.0.1:33259, datanodeUuid=76be8b2a-67e3-435a-b5f7-adea186d1f20, infoPort=35803, infoSecurePort=0, ipcPort=35437, storageInfo=lv=-57;cid=testClusterID;nsid=1702142786;c=1732776821123), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-28T06:53:43,814 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x65b9b11602ead7e3 with lease ID 0x50e443b2445aac48: Processing first storage report for DS-dab0c4ad-deb5-4e38-a2a3-6a81b7f42af6 from datanode DatanodeRegistration(127.0.0.1:33259, datanodeUuid=76be8b2a-67e3-435a-b5f7-adea186d1f20, infoPort=35803, infoSecurePort=0, ipcPort=35437, storageInfo=lv=-57;cid=testClusterID;nsid=1702142786;c=1732776821123) 2024-11-28T06:53:43,814 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x65b9b11602ead7e3 with lease ID 0x50e443b2445aac48: from storage DS-dab0c4ad-deb5-4e38-a2a3-6a81b7f42af6 node DatanodeRegistration(127.0.0.1:33259, datanodeUuid=76be8b2a-67e3-435a-b5f7-adea186d1f20, infoPort=35803, infoSecurePort=0, ipcPort=35437, storageInfo=lv=-57;cid=testClusterID;nsid=1702142786;c=1732776821123), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-28T06:53:43,871 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ce3257b-8be6-c2e3-213a-03473f81f2f4 2024-11-28T06:53:43,945 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ce3257b-8be6-c2e3-213a-03473f81f2f4/cluster_b8990226-b232-1028-e659-af5834b9be30/zookeeper_0, clientPort=51984, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ce3257b-8be6-c2e3-213a-03473f81f2f4/cluster_b8990226-b232-1028-e659-af5834b9be30/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ce3257b-8be6-c2e3-213a-03473f81f2f4/cluster_b8990226-b232-1028-e659-af5834b9be30/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-28T06:53:43,954 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51984 2024-11-28T06:53:43,969 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T06:53:43,973 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T06:53:44,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741825_1001 (size=7) 2024-11-28T06:53:44,602 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7 with version=8 2024-11-28T06:53:44,602 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/hbase-staging 2024-11-28T06:53:44,727 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-28T06:53:44,982 INFO [Time-limited test {}] client.ConnectionUtils(128): master/1cb9cf437a36:0 server-side Connection retries=45 2024-11-28T06:53:44,991 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T06:53:44,991 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T06:53:44,996 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T06:53:44,996 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T06:53:44,996 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T06:53:45,120 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-28T06:53:45,171 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-28T06:53:45,180 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-28T06:53:45,183 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T06:53:45,204 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 104748 (auto-detected) 2024-11-28T06:53:45,205 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-28T06:53:45,222 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37503 2024-11-28T06:53:45,241 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37503 connecting to ZooKeeper ensemble=127.0.0.1:51984 2024-11-28T06:53:45,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:375030x0, quorum=127.0.0.1:51984, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T06:53:45,363 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37503-0x10181819f390000 connected 2024-11-28T06:53:45,441 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T06:53:45,443 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T06:53:45,461 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37503-0x10181819f390000, quorum=127.0.0.1:51984, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T06:53:45,466 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7, hbase.cluster.distributed=false 2024-11-28T06:53:45,501 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37503-0x10181819f390000, quorum=127.0.0.1:51984, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T06:53:45,506 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37503 2024-11-28T06:53:45,507 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37503 2024-11-28T06:53:45,507 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37503 2024-11-28T06:53:45,508 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37503 2024-11-28T06:53:45,508 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37503 2024-11-28T06:53:45,606 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1cb9cf437a36:0 server-side Connection retries=45 2024-11-28T06:53:45,608 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T06:53:45,608 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T06:53:45,608 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T06:53:45,608 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T06:53:45,609 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T06:53:45,612 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-28T06:53:45,615 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T06:53:45,623 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43341 2024-11-28T06:53:45,626 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43341 connecting to ZooKeeper ensemble=127.0.0.1:51984 2024-11-28T06:53:45,629 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T06:53:45,633 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T06:53:45,656 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:433410x0, quorum=127.0.0.1:51984, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T06:53:45,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:433410x0, quorum=127.0.0.1:51984, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T06:53:45,664 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-28T06:53:45,665 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43341-0x10181819f390001 connected 2024-11-28T06:53:45,682 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-28T06:53:45,686 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43341-0x10181819f390001, quorum=127.0.0.1:51984, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-28T06:53:45,692 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43341-0x10181819f390001, quorum=127.0.0.1:51984, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T06:53:45,694 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43341 2024-11-28T06:53:45,695 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43341 2024-11-28T06:53:45,707 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43341 2024-11-28T06:53:45,709 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43341 2024-11-28T06:53:45,711 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43341 2024-11-28T06:53:45,737 DEBUG [M:0;1cb9cf437a36:37503 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;1cb9cf437a36:37503 2024-11-28T06:53:45,738 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/1cb9cf437a36,37503,1732776824789 2024-11-28T06:53:45,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37503-0x10181819f390000, quorum=127.0.0.1:51984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T06:53:45,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43341-0x10181819f390001, quorum=127.0.0.1:51984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T06:53:45,764 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37503-0x10181819f390000, quorum=127.0.0.1:51984, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/1cb9cf437a36,37503,1732776824789 2024-11-28T06:53:45,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37503-0x10181819f390000, quorum=127.0.0.1:51984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:53:45,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43341-0x10181819f390001, quorum=127.0.0.1:51984, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-28T06:53:45,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43341-0x10181819f390001, quorum=127.0.0.1:51984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:53:45,797 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37503-0x10181819f390000, quorum=127.0.0.1:51984, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-28T06:53:45,799 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/1cb9cf437a36,37503,1732776824789 from backup master directory 2024-11-28T06:53:45,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37503-0x10181819f390000, quorum=127.0.0.1:51984, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/1cb9cf437a36,37503,1732776824789 2024-11-28T06:53:45,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43341-0x10181819f390001, quorum=127.0.0.1:51984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T06:53:45,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37503-0x10181819f390000, quorum=127.0.0.1:51984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T06:53:45,812 WARN [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T06:53:45,813 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=1cb9cf437a36,37503,1732776824789 2024-11-28T06:53:45,815 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-28T06:53:45,816 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-28T06:53:45,870 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/hbase.id] with ID: d2033d23-da74-47d8-82e3-5e5da01454bd 2024-11-28T06:53:45,870 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/.tmp/hbase.id 2024-11-28T06:53:45,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741826_1002 (size=42) 2024-11-28T06:53:46,285 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/.tmp/hbase.id]:[hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/hbase.id] 2024-11-28T06:53:46,334 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T06:53:46,340 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-28T06:53:46,358 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-28T06:53:46,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43341-0x10181819f390001, quorum=127.0.0.1:51984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:53:46,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37503-0x10181819f390000, quorum=127.0.0.1:51984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:53:46,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741827_1003 (size=196) 2024-11-28T06:53:46,808 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T06:53:46,810 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-28T06:53:46,828 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:46,833 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T06:53:46,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741828_1004 (size=1189) 2024-11-28T06:53:47,293 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/MasterData/data/master/store 2024-11-28T06:53:47,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741829_1005 (size=34) 2024-11-28T06:53:47,718 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-28T06:53:47,721 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T06:53:47,722 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-28T06:53:47,722 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T06:53:47,723 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T06:53:47,724 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-28T06:53:47,724 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T06:53:47,724 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T06:53:47,725 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732776827722Disabling compacts and flushes for region at 1732776827722Disabling writes for close at 1732776827724 (+2 ms)Writing region close event to WAL at 1732776827724Closed at 1732776827724 2024-11-28T06:53:47,728 WARN [master/1cb9cf437a36:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/MasterData/data/master/store/.initializing 2024-11-28T06:53:47,728 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/MasterData/WALs/1cb9cf437a36,37503,1732776824789 2024-11-28T06:53:47,738 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T06:53:47,758 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1cb9cf437a36%2C37503%2C1732776824789, suffix=, logDir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/MasterData/WALs/1cb9cf437a36,37503,1732776824789, archiveDir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/MasterData/oldWALs, maxLogs=10 2024-11-28T06:53:47,785 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/MasterData/WALs/1cb9cf437a36,37503,1732776824789/1cb9cf437a36%2C37503%2C1732776824789.1732776827763, exclude list is [], retry=0 2024-11-28T06:53:47,807 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33259,DS-668c083b-bcb6-429c-8335-603571730fb1,DISK] 2024-11-28T06:53:47,810 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-28T06:53:47,852 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/MasterData/WALs/1cb9cf437a36,37503,1732776824789/1cb9cf437a36%2C37503%2C1732776824789.1732776827763 2024-11-28T06:53:47,854 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35803:35803)] 2024-11-28T06:53:47,854 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-28T06:53:47,855 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T06:53:47,859 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T06:53:47,860 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T06:53:47,900 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T06:53:47,926 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-28T06:53:47,929 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:53:47,932 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T06:53:47,933 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T06:53:47,936 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-28T06:53:47,937 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:53:47,938 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:53:47,938 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T06:53:47,941 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-28T06:53:47,942 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:53:47,943 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:53:47,943 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T06:53:47,947 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-28T06:53:47,947 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:53:47,948 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:53:47,949 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T06:53:47,952 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-28T06:53:47,954 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-28T06:53:47,959 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T06:53:47,960 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T06:53:47,964 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T06:53:47,969 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T06:53:47,973 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T06:53:47,975 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63652108, jitterRate=-0.05150967836380005}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T06:53:47,981 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732776827873Initializing all the Stores at 1732776827876 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732776827877 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732776827878 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732776827878Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732776827878Cleaning up temporary data from old regions at 1732776827960 (+82 ms)Region opened successfully at 1732776827981 (+21 ms) 2024-11-28T06:53:47,983 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-28T06:53:48,029 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@521cbb37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1cb9cf437a36/172.17.0.2:0 2024-11-28T06:53:48,071 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-28T06:53:48,085 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-28T06:53:48,085 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-28T06:53:48,089 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T06:53:48,091 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-28T06:53:48,098 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-28T06:53:48,099 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-28T06:53:48,136 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-28T06:53:48,149 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37503-0x10181819f390000, quorum=127.0.0.1:51984, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-28T06:53:48,204 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-28T06:53:48,207 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-28T06:53:48,209 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37503-0x10181819f390000, quorum=127.0.0.1:51984, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-28T06:53:48,221 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-28T06:53:48,224 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-28T06:53:48,229 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37503-0x10181819f390000, quorum=127.0.0.1:51984, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-28T06:53:48,237 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-28T06:53:48,239 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37503-0x10181819f390000, quorum=127.0.0.1:51984, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-28T06:53:48,245 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-28T06:53:48,267 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37503-0x10181819f390000, quorum=127.0.0.1:51984, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-28T06:53:48,279 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-28T06:53:48,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43341-0x10181819f390001, quorum=127.0.0.1:51984, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-28T06:53:48,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37503-0x10181819f390000, quorum=127.0.0.1:51984, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-28T06:53:48,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43341-0x10181819f390001, quorum=127.0.0.1:51984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:53:48,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37503-0x10181819f390000, quorum=127.0.0.1:51984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:53:48,291 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=1cb9cf437a36,37503,1732776824789, sessionid=0x10181819f390000, setting cluster-up flag (Was=false) 2024-11-28T06:53:48,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43341-0x10181819f390001, quorum=127.0.0.1:51984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:53:48,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37503-0x10181819f390000, quorum=127.0.0.1:51984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:53:48,346 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-28T06:53:48,348 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1cb9cf437a36,37503,1732776824789 2024-11-28T06:53:48,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43341-0x10181819f390001, quorum=127.0.0.1:51984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:53:48,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37503-0x10181819f390000, quorum=127.0.0.1:51984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:53:48,395 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-28T06:53:48,398 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1cb9cf437a36,37503,1732776824789 2024-11-28T06:53:48,405 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-28T06:53:48,420 INFO [RS:0;1cb9cf437a36:43341 {}] regionserver.HRegionServer(746): ClusterId : d2033d23-da74-47d8-82e3-5e5da01454bd 2024-11-28T06:53:48,422 DEBUG [RS:0;1cb9cf437a36:43341 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-28T06:53:48,431 DEBUG [RS:0;1cb9cf437a36:43341 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-28T06:53:48,431 DEBUG [RS:0;1cb9cf437a36:43341 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-28T06:53:48,438 DEBUG [RS:0;1cb9cf437a36:43341 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-28T06:53:48,439 DEBUG [RS:0;1cb9cf437a36:43341 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@271a34a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1cb9cf437a36/172.17.0.2:0 2024-11-28T06:53:48,452 DEBUG [RS:0;1cb9cf437a36:43341 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;1cb9cf437a36:43341 2024-11-28T06:53:48,455 INFO [RS:0;1cb9cf437a36:43341 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-28T06:53:48,456 INFO [RS:0;1cb9cf437a36:43341 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-28T06:53:48,456 DEBUG [RS:0;1cb9cf437a36:43341 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-28T06:53:48,459 INFO [RS:0;1cb9cf437a36:43341 {}] regionserver.HRegionServer(2659): reportForDuty to master=1cb9cf437a36,37503,1732776824789 with port=43341, startcode=1732776825568 2024-11-28T06:53:48,473 DEBUG [RS:0;1cb9cf437a36:43341 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-28T06:53:48,484 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T06:53:48,497 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-28T06:53:48,508 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-28T06:53:48,515 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 1cb9cf437a36,37503,1732776824789 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-28T06:53:48,524 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45803, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-28T06:53:48,525 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/1cb9cf437a36:0, corePoolSize=5, maxPoolSize=5 2024-11-28T06:53:48,525 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/1cb9cf437a36:0, corePoolSize=5, maxPoolSize=5 2024-11-28T06:53:48,525 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/1cb9cf437a36:0, corePoolSize=5, maxPoolSize=5 2024-11-28T06:53:48,525 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/1cb9cf437a36:0, corePoolSize=5, maxPoolSize=5 2024-11-28T06:53:48,526 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/1cb9cf437a36:0, corePoolSize=10, maxPoolSize=10 2024-11-28T06:53:48,526 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:53:48,526 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/1cb9cf437a36:0, corePoolSize=2, maxPoolSize=2 2024-11-28T06:53:48,527 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:53:48,531 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37503 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:48,545 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732776858545 2024-11-28T06:53:48,549 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-28T06:53:48,551 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-28T06:53:48,551 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T06:53:48,552 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-28T06:53:48,560 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:53:48,560 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-28T06:53:48,566 DEBUG [RS:0;1cb9cf437a36:43341 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-28T06:53:48,566 WARN [RS:0;1cb9cf437a36:43341 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-28T06:53:48,569 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-28T06:53:48,569 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-28T06:53:48,570 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-28T06:53:48,570 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-28T06:53:48,575 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-28T06:53:48,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741831_1007 (size=1321) 2024-11-28T06:53:48,587 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-28T06:53:48,589 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-28T06:53:48,590 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-28T06:53:48,590 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7 2024-11-28T06:53:48,591 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-28T06:53:48,595 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-28T06:53:48,595 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-28T06:53:48,600 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/1cb9cf437a36:0:becomeActiveMaster-HFileCleaner.large.0-1732776828597,5,FailOnTimeoutGroup] 2024-11-28T06:53:48,601 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/1cb9cf437a36:0:becomeActiveMaster-HFileCleaner.small.0-1732776828600,5,FailOnTimeoutGroup] 2024-11-28T06:53:48,601 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-28T06:53:48,602 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-28T06:53:48,603 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-28T06:53:48,604 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-28T06:53:48,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741832_1008 (size=32) 2024-11-28T06:53:48,668 INFO [RS:0;1cb9cf437a36:43341 {}] regionserver.HRegionServer(2659): reportForDuty to master=1cb9cf437a36,37503,1732776824789 with port=43341, startcode=1732776825568 2024-11-28T06:53:48,670 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37503 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:48,672 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37503 {}] master.ServerManager(517): Registering regionserver=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:48,680 DEBUG [RS:0;1cb9cf437a36:43341 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7 2024-11-28T06:53:48,680 DEBUG [RS:0;1cb9cf437a36:43341 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38743 2024-11-28T06:53:48,680 DEBUG [RS:0;1cb9cf437a36:43341 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-28T06:53:48,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37503-0x10181819f390000, quorum=127.0.0.1:51984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T06:53:48,712 DEBUG [RS:0;1cb9cf437a36:43341 {}] zookeeper.ZKUtil(111): regionserver:43341-0x10181819f390001, quorum=127.0.0.1:51984, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:48,712 WARN [RS:0;1cb9cf437a36:43341 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T06:53:48,713 INFO [RS:0;1cb9cf437a36:43341 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T06:53:48,713 DEBUG [RS:0;1cb9cf437a36:43341 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/WALs/1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:48,714 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1cb9cf437a36,43341,1732776825568] 2024-11-28T06:53:48,744 INFO [RS:0;1cb9cf437a36:43341 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-28T06:53:48,764 INFO [RS:0;1cb9cf437a36:43341 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-28T06:53:48,771 INFO [RS:0;1cb9cf437a36:43341 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-28T06:53:48,771 INFO [RS:0;1cb9cf437a36:43341 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T06:53:48,773 INFO [RS:0;1cb9cf437a36:43341 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-28T06:53:48,781 INFO [RS:0;1cb9cf437a36:43341 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-28T06:53:48,783 INFO [RS:0;1cb9cf437a36:43341 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-28T06:53:48,784 DEBUG [RS:0;1cb9cf437a36:43341 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:53:48,784 DEBUG [RS:0;1cb9cf437a36:43341 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:53:48,784 DEBUG [RS:0;1cb9cf437a36:43341 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:53:48,784 DEBUG [RS:0;1cb9cf437a36:43341 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:53:48,784 DEBUG [RS:0;1cb9cf437a36:43341 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:53:48,785 DEBUG [RS:0;1cb9cf437a36:43341 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0, corePoolSize=2, maxPoolSize=2 2024-11-28T06:53:48,785 DEBUG [RS:0;1cb9cf437a36:43341 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:53:48,785 DEBUG [RS:0;1cb9cf437a36:43341 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:53:48,785 DEBUG [RS:0;1cb9cf437a36:43341 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:53:48,785 DEBUG [RS:0;1cb9cf437a36:43341 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:53:48,785 DEBUG [RS:0;1cb9cf437a36:43341 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:53:48,786 DEBUG [RS:0;1cb9cf437a36:43341 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:53:48,786 DEBUG [RS:0;1cb9cf437a36:43341 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1cb9cf437a36:0, corePoolSize=3, maxPoolSize=3 2024-11-28T06:53:48,786 DEBUG [RS:0;1cb9cf437a36:43341 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0, corePoolSize=3, maxPoolSize=3 2024-11-28T06:53:48,788 INFO [RS:0;1cb9cf437a36:43341 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T06:53:48,788 INFO [RS:0;1cb9cf437a36:43341 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T06:53:48,788 INFO [RS:0;1cb9cf437a36:43341 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T06:53:48,788 INFO [RS:0;1cb9cf437a36:43341 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-28T06:53:48,788 INFO [RS:0;1cb9cf437a36:43341 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-28T06:53:48,788 INFO [RS:0;1cb9cf437a36:43341 {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,43341,1732776825568-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T06:53:48,811 INFO [RS:0;1cb9cf437a36:43341 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-28T06:53:48,813 INFO [RS:0;1cb9cf437a36:43341 {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,43341,1732776825568-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T06:53:48,813 INFO [RS:0;1cb9cf437a36:43341 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T06:53:48,814 INFO [RS:0;1cb9cf437a36:43341 {}] regionserver.Replication(171): 1cb9cf437a36,43341,1732776825568 started 2024-11-28T06:53:48,837 INFO [RS:0;1cb9cf437a36:43341 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T06:53:48,838 INFO [RS:0;1cb9cf437a36:43341 {}] regionserver.HRegionServer(1482): Serving as 1cb9cf437a36,43341,1732776825568, RpcServer on 1cb9cf437a36/172.17.0.2:43341, sessionid=0x10181819f390001 2024-11-28T06:53:48,839 DEBUG [RS:0;1cb9cf437a36:43341 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-28T06:53:48,839 DEBUG [RS:0;1cb9cf437a36:43341 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:48,839 DEBUG [RS:0;1cb9cf437a36:43341 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1cb9cf437a36,43341,1732776825568' 2024-11-28T06:53:48,840 DEBUG [RS:0;1cb9cf437a36:43341 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-28T06:53:48,841 DEBUG [RS:0;1cb9cf437a36:43341 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-28T06:53:48,842 DEBUG [RS:0;1cb9cf437a36:43341 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-28T06:53:48,842 DEBUG [RS:0;1cb9cf437a36:43341 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-28T06:53:48,842 DEBUG [RS:0;1cb9cf437a36:43341 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:48,842 DEBUG [RS:0;1cb9cf437a36:43341 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1cb9cf437a36,43341,1732776825568' 2024-11-28T06:53:48,842 DEBUG [RS:0;1cb9cf437a36:43341 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-28T06:53:48,843 DEBUG [RS:0;1cb9cf437a36:43341 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-28T06:53:48,844 DEBUG [RS:0;1cb9cf437a36:43341 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-28T06:53:48,844 INFO [RS:0;1cb9cf437a36:43341 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-28T06:53:48,845 INFO [RS:0;1cb9cf437a36:43341 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-28T06:53:48,949 INFO [RS:0;1cb9cf437a36:43341 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T06:53:48,953 INFO [RS:0;1cb9cf437a36:43341 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1cb9cf437a36%2C43341%2C1732776825568, suffix=, logDir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/WALs/1cb9cf437a36,43341,1732776825568, archiveDir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/oldWALs, maxLogs=32 2024-11-28T06:53:48,976 DEBUG [RS:0;1cb9cf437a36:43341 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/WALs/1cb9cf437a36,43341,1732776825568/1cb9cf437a36%2C43341%2C1732776825568.1732776828956, exclude list is [], retry=0 2024-11-28T06:53:48,980 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33259,DS-668c083b-bcb6-429c-8335-603571730fb1,DISK] 2024-11-28T06:53:48,984 INFO [RS:0;1cb9cf437a36:43341 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/WALs/1cb9cf437a36,43341,1732776825568/1cb9cf437a36%2C43341%2C1732776825568.1732776828956 2024-11-28T06:53:48,987 DEBUG [RS:0;1cb9cf437a36:43341 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35803:35803)] 2024-11-28T06:53:49,020 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T06:53:49,022 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-28T06:53:49,025 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-28T06:53:49,025 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:53:49,026 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T06:53:49,026 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-28T06:53:49,028 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-28T06:53:49,029 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:53:49,029 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T06:53:49,030 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-28T06:53:49,032 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-28T06:53:49,032 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:53:49,033 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T06:53:49,033 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-28T06:53:49,036 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-28T06:53:49,036 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:53:49,037 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T06:53:49,037 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-28T06:53:49,039 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/hbase/meta/1588230740 2024-11-28T06:53:49,040 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/hbase/meta/1588230740 2024-11-28T06:53:49,043 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-28T06:53:49,043 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-28T06:53:49,044 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T06:53:49,047 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-28T06:53:49,051 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T06:53:49,052 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68021778, jitterRate=0.01360347867012024}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T06:53:49,054 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732776829020Initializing all the Stores at 1732776829022 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732776829022Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732776829022Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732776829022Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732776829022Cleaning up temporary data from old regions at 1732776829043 (+21 ms)Region opened successfully at 1732776829054 (+11 ms) 2024-11-28T06:53:49,055 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-28T06:53:49,055 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-28T06:53:49,055 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-28T06:53:49,055 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-28T06:53:49,055 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-28T06:53:49,057 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-28T06:53:49,057 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732776829055Disabling compacts and flushes for region at 1732776829055Disabling writes for close at 1732776829055Writing region close event to WAL at 1732776829056 (+1 ms)Closed at 1732776829057 (+1 ms) 2024-11-28T06:53:49,061 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T06:53:49,061 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-28T06:53:49,067 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-28T06:53:49,075 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T06:53:49,077 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-28T06:53:49,230 DEBUG [1cb9cf437a36:37503 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-28T06:53:49,242 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:49,248 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1cb9cf437a36,43341,1732776825568, state=OPENING 2024-11-28T06:53:49,312 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-28T06:53:49,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43341-0x10181819f390001, quorum=127.0.0.1:51984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:53:49,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37503-0x10181819f390000, quorum=127.0.0.1:51984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:53:49,321 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T06:53:49,323 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T06:53:49,323 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T06:53:49,326 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,43341,1732776825568}] 2024-11-28T06:53:49,507 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-28T06:53:49,512 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45677, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-28T06:53:49,525 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-28T06:53:49,526 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T06:53:49,527 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-28T06:53:49,531 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1cb9cf437a36%2C43341%2C1732776825568.meta, suffix=.meta, logDir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/WALs/1cb9cf437a36,43341,1732776825568, archiveDir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/oldWALs, maxLogs=32 2024-11-28T06:53:49,554 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/WALs/1cb9cf437a36,43341,1732776825568/1cb9cf437a36%2C43341%2C1732776825568.meta.1732776829534.meta, exclude list is [], retry=0 2024-11-28T06:53:49,558 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33259,DS-668c083b-bcb6-429c-8335-603571730fb1,DISK] 2024-11-28T06:53:49,562 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/WALs/1cb9cf437a36,43341,1732776825568/1cb9cf437a36%2C43341%2C1732776825568.meta.1732776829534.meta 2024-11-28T06:53:49,563 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35803:35803)] 2024-11-28T06:53:49,563 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-28T06:53:49,565 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-28T06:53:49,568 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-28T06:53:49,574 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-28T06:53:49,577 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-28T06:53:49,578 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T06:53:49,578 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-28T06:53:49,578 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-28T06:53:49,581 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-28T06:53:49,583 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-28T06:53:49,583 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:53:49,584 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T06:53:49,584 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-28T06:53:49,586 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-28T06:53:49,586 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:53:49,587 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T06:53:49,587 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-28T06:53:49,589 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-28T06:53:49,589 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:53:49,590 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T06:53:49,590 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-28T06:53:49,592 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-28T06:53:49,592 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:53:49,593 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T06:53:49,593 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-28T06:53:49,595 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/hbase/meta/1588230740 2024-11-28T06:53:49,597 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/hbase/meta/1588230740 2024-11-28T06:53:49,600 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-28T06:53:49,600 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-28T06:53:49,601 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T06:53:49,605 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-28T06:53:49,607 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68698266, jitterRate=0.023683935403823853}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T06:53:49,607 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-28T06:53:49,609 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732776829579Writing region info on filesystem at 1732776829579Initializing all the Stores at 1732776829581 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732776829581Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732776829581Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732776829581Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732776829581Cleaning up temporary data from old regions at 1732776829600 (+19 ms)Running coprocessor post-open hooks at 1732776829607 (+7 ms)Region opened successfully at 1732776829609 (+2 ms) 2024-11-28T06:53:49,616 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732776829497 2024-11-28T06:53:49,629 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-28T06:53:49,629 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-28T06:53:49,630 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:49,632 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1cb9cf437a36,43341,1732776825568, state=OPEN 2024-11-28T06:53:49,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43341-0x10181819f390001, quorum=127.0.0.1:51984, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T06:53:49,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37503-0x10181819f390000, quorum=127.0.0.1:51984, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T06:53:49,725 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T06:53:49,725 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T06:53:49,726 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:49,732 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-28T06:53:49,732 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,43341,1732776825568 in 400 msec 2024-11-28T06:53:49,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-28T06:53:49,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 667 msec 2024-11-28T06:53:49,742 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T06:53:49,742 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-28T06:53:49,763 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:53:49,765 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:53:49,790 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:53:49,793 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41039, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:53:49,815 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.3790 sec 2024-11-28T06:53:49,815 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732776829815, completionTime=-1 2024-11-28T06:53:49,818 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-28T06:53:49,818 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T06:53:49,846 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-28T06:53:49,847 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732776889847 2024-11-28T06:53:49,847 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732776949847 2024-11-28T06:53:49,847 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 28 msec 2024-11-28T06:53:49,859 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,37503,1732776824789-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T06:53:49,859 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,37503,1732776824789-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T06:53:49,859 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,37503,1732776824789-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T06:53:49,861 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-1cb9cf437a36:37503, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T06:53:49,861 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-28T06:53:49,862 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-28T06:53:49,868 DEBUG [master/1cb9cf437a36:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-28T06:53:49,890 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.077sec 2024-11-28T06:53:49,892 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-28T06:53:49,893 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-28T06:53:49,894 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-28T06:53:49,894 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-28T06:53:49,895 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-28T06:53:49,895 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,37503,1732776824789-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T06:53:49,896 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,37503,1732776824789-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-28T06:53:49,904 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-28T06:53:49,905 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-28T06:53:49,905 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,37503,1732776824789-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T06:53:49,932 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:53:49,933 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-28T06:53:49,934 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-28T06:53:49,936 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:53:49,939 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:53:49,947 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:53:49,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:53:49,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:53:49,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61ae9ae4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:53:49,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:53:49,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:53:49,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:53:49,958 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41654, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:53:49,961 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:53:49,961 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:53:49,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:53:49,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:53:49,972 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:53:49,993 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=1cb9cf437a36,37503,1732776824789 2024-11-28T06:53:50,023 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=389, ProcessCount=11, AvailableMemoryMB=9155 2024-11-28T06:53:50,035 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T06:53:50,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 1cb9cf437a36,37503,1732776824789 2024-11-28T06:53:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@13dda2cd 2024-11-28T06:53:50,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T06:53:50,051 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41658, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T06:53:50,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T06:53:50,062 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T06:53:50,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-28T06:53:50,074 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T06:53:50,077 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-28T06:53:50,077 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:53:50,080 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T06:53:50,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T06:53:50,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741835_1011 (size=1061) 2024-11-28T06:53:50,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T06:53:50,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T06:53:50,497 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7 2024-11-28T06:53:50,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741836_1012 (size=53) 2024-11-28T06:53:50,508 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T06:53:50,508 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d6ca3b1ced864db1c50bb8f904a838f3, disabling compactions & flushes 2024-11-28T06:53:50,508 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:50,508 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:50,508 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. after waiting 0 ms 2024-11-28T06:53:50,508 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:50,509 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:50,509 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d6ca3b1ced864db1c50bb8f904a838f3: Waiting for close lock at 1732776830508Disabling compacts and flushes for region at 1732776830508Disabling writes for close at 1732776830508Writing region close event to WAL at 1732776830508Closed at 1732776830508 2024-11-28T06:53:50,512 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T06:53:50,517 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732776830512"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732776830512"}]},"ts":"1732776830512"} 2024-11-28T06:53:50,525 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T06:53:50,528 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T06:53:50,532 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732776830529"}]},"ts":"1732776830529"} 2024-11-28T06:53:50,537 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-28T06:53:50,539 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d6ca3b1ced864db1c50bb8f904a838f3, ASSIGN}] 2024-11-28T06:53:50,542 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d6ca3b1ced864db1c50bb8f904a838f3, ASSIGN 2024-11-28T06:53:50,544 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d6ca3b1ced864db1c50bb8f904a838f3, ASSIGN; state=OFFLINE, location=1cb9cf437a36,43341,1732776825568; forceNewPlan=false, retain=false 2024-11-28T06:53:50,696 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d6ca3b1ced864db1c50bb8f904a838f3, regionState=OPENING, regionLocation=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:50,701 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d6ca3b1ced864db1c50bb8f904a838f3, ASSIGN because future has completed 2024-11-28T06:53:50,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568}] 2024-11-28T06:53:50,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T06:53:50,865 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:50,865 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} 2024-11-28T06:53:50,866 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:53:50,866 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T06:53:50,866 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:53:50,867 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:53:50,873 INFO [StoreOpener-d6ca3b1ced864db1c50bb8f904a838f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:53:50,878 INFO [StoreOpener-d6ca3b1ced864db1c50bb8f904a838f3-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T06:53:50,879 INFO [StoreOpener-d6ca3b1ced864db1c50bb8f904a838f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d6ca3b1ced864db1c50bb8f904a838f3 columnFamilyName A 2024-11-28T06:53:50,879 DEBUG [StoreOpener-d6ca3b1ced864db1c50bb8f904a838f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:53:50,881 INFO [StoreOpener-d6ca3b1ced864db1c50bb8f904a838f3-1 {}] regionserver.HStore(327): Store=d6ca3b1ced864db1c50bb8f904a838f3/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:53:50,881 INFO [StoreOpener-d6ca3b1ced864db1c50bb8f904a838f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:53:50,884 INFO [StoreOpener-d6ca3b1ced864db1c50bb8f904a838f3-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T06:53:50,885 INFO [StoreOpener-d6ca3b1ced864db1c50bb8f904a838f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d6ca3b1ced864db1c50bb8f904a838f3 columnFamilyName B 2024-11-28T06:53:50,885 DEBUG [StoreOpener-d6ca3b1ced864db1c50bb8f904a838f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:53:50,886 INFO [StoreOpener-d6ca3b1ced864db1c50bb8f904a838f3-1 {}] regionserver.HStore(327): Store=d6ca3b1ced864db1c50bb8f904a838f3/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:53:50,887 INFO [StoreOpener-d6ca3b1ced864db1c50bb8f904a838f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:53:50,889 INFO [StoreOpener-d6ca3b1ced864db1c50bb8f904a838f3-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T06:53:50,890 INFO [StoreOpener-d6ca3b1ced864db1c50bb8f904a838f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d6ca3b1ced864db1c50bb8f904a838f3 columnFamilyName C 2024-11-28T06:53:50,890 DEBUG [StoreOpener-d6ca3b1ced864db1c50bb8f904a838f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:53:50,891 INFO [StoreOpener-d6ca3b1ced864db1c50bb8f904a838f3-1 {}] regionserver.HStore(327): Store=d6ca3b1ced864db1c50bb8f904a838f3/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:53:50,895 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:50,895 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:53:50,897 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:53:50,898 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:53:50,901 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:53:50,901 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:53:50,903 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T06:53:50,907 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:53:50,917 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T06:53:50,919 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened d6ca3b1ced864db1c50bb8f904a838f3; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74241597, jitterRate=0.10628600418567657}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T06:53:50,919 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:53:50,920 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for d6ca3b1ced864db1c50bb8f904a838f3: Running coprocessor pre-open hook at 1732776830867Writing region info on filesystem at 1732776830867Initializing all the Stores at 1732776830869 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732776830869Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732776830873 (+4 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732776830873Cleaning up temporary data from old regions at 1732776830901 (+28 ms)Running coprocessor post-open hooks at 1732776830919 (+18 ms)Region opened successfully at 1732776830920 (+1 ms) 2024-11-28T06:53:50,923 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., pid=6, masterSystemTime=1732776830858 2024-11-28T06:53:50,928 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:50,928 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:50,930 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d6ca3b1ced864db1c50bb8f904a838f3, regionState=OPEN, openSeqNum=2, regionLocation=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:50,939 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 because future has completed 2024-11-28T06:53:50,950 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-28T06:53:50,950 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 in 238 msec 2024-11-28T06:53:50,956 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-28T06:53:50,956 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d6ca3b1ced864db1c50bb8f904a838f3, ASSIGN in 412 msec 2024-11-28T06:53:50,958 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T06:53:50,959 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732776830958"}]},"ts":"1732776830958"} 2024-11-28T06:53:50,963 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-28T06:53:50,965 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T06:53:50,970 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 899 msec 2024-11-28T06:53:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T06:53:51,227 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:53:51,234 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:53:51,234 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:53:51,234 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:53:51,236 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:53:51,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:53:51,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:53:51,238 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@703ed473, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:53:51,238 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:53:51,238 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:53:51,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:53:51,240 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41670, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:53:51,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a97401f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:53:51,243 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:53:51,245 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:53:51,245 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:53:51,248 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36842, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:53:51,251 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T06:53:51,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 1cb9cf437a36,37503,1732776824789 2024-11-28T06:53:51,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@37b6628a 2024-11-28T06:53:51,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T06:53:51,259 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41676, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T06:53:51,265 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:53:51,265 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:53:51,265 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:53:51,267 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:53:51,267 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:53:51,268 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:53:51,268 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e45ede1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:53:51,268 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:53:51,269 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:53:51,272 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:53:51,273 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41702, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:53:51,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d54f318, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:53:51,276 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:53:51,277 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:53:51,277 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:53:51,279 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:53:51,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:53:51,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:53:51,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1793d55e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:53:51,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:53:51,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:53:51,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:53:51,284 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41714, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:53:51,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@357c1d7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:53:51,292 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:53:51,292 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:53:51,292 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:53:51,295 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:53:51,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:53:51,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:53:51,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67aa5cce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:53:51,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:53:51,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:53:51,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:53:51,299 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41740, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:53:51,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61f1a5be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:53:51,302 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:53:51,302 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:53:51,302 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:53:51,304 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:53:51,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:53:51,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:53:51,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b25cfff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:53:51,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:53:51,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:53:51,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:53:51,309 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41770, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:53:51,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@478b81ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:53:51,312 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:53:51,312 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:53:51,312 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:53:51,313 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:53:51,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:53:51,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:53:51,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@84b2157, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:53:51,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:53:51,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:53:51,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:53:51,317 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41776, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:53:51,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18f2bd56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:53:51,321 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:53:51,321 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:53:51,321 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:53:51,322 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:53:51,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:53:51,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:53:51,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65f07fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:53:51,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:53:51,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:53:51,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:53:51,326 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41806, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:53:51,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f43484c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:53:51,331 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:53:51,331 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:53:51,331 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:53:51,333 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:53:51,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:53:51,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:53:51,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c83f8e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:53:51,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:53:51,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:53:51,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:53:51,337 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41816, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:53:51,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77963136, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:53:51,342 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:53:51,342 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:53:51,343 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:53:51,344 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:53:51,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:53:51,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:53:51,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35159008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:53:51,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:53:51,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:53:51,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:53:51,348 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41840, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:53:51,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@149eeca3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:53:51,354 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:53:51,355 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:53:51,355 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:53:51,357 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:53:51,358 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:53:51,358 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:53:51,358 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1567aa3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:53:51,358 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:53:51,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:53:51,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:53:51,361 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41862, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:53:51,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18830d5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:53:51,376 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:53:51,376 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:53:51,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:53:51,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:53:51,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:53:51,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:53:51,381 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36856, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:53:51,388 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36870, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:53:51,388 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:53:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:53:51,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2] 2024-11-28T06:53:51,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:53:51,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2] 2024-11-28T06:53:51,394 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36884, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:53:51,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2] 2024-11-28T06:53:51,408 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:53:51,410 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:53:51,411 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:53:51,421 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36896, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:53:51,423 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:53:51,424 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:53:51,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2] 2024-11-28T06:53:51,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:53:51,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:53:51,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:53:51,430 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36906, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:53:51,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:53:51,435 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:53:51,436 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36922, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:53:51,437 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:53:51,438 DEBUG [Thread-151 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:53:51,439 DEBUG [Thread-151 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:53:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:53:51,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:53:51,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2] 2024-11-28T06:53:51,442 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36932, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:53:51,442 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36946, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:53:51,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2] 2024-11-28T06:53:51,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2] 2024-11-28T06:53:51,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2] 2024-11-28T06:53:51,452 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:53:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-28T06:53:51,462 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:53:51,465 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:53:51,467 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:53:51,469 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:53:51,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:53:51,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:53:51,474 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36956, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:53:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T06:53:51,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2] 2024-11-28T06:53:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:53:51,497 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6ca3b1ced864db1c50bb8f904a838f3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T06:53:51,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=A 2024-11-28T06:53:51,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:53:51,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=B 2024-11-28T06:53:51,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:53:51,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=C 2024-11-28T06:53:51,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:53:51,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T06:53:51,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/40db68802f584dd8afa8de305e7cd21b is 50, key is test_row_0/A:col10/1732776831486/Put/seqid=0 2024-11-28T06:53:51,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T06:53:51,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776891632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:51,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36870 deadline: 1732776891632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:51,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:51,696 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:51,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:51,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:51,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:51,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:51,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776891641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:51,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776891663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:51,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:51,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:51,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:51,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:51,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:51,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:51,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741837_1013 (size=14341) 2024-11-28T06:53:51,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776891687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:51,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:51,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:51,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:51,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/40db68802f584dd8afa8de305e7cd21b 2024-11-28T06:53:51,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:51,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:53:51,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:51,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:51,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:51,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:51,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T06:53:51,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:51,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36870 deadline: 1732776891808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:51,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:51,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:51,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:51,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:51,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776891809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:51,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:51,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:51,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:51,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:51,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776891817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:51,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:51,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776891818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:51,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:51,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776891818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:51,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:51,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/99de9edc2c644e52a35fb298627b045e is 50, key is test_row_0/B:col10/1732776831486/Put/seqid=0 2024-11-28T06:53:51,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:51,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:51,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:51,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:51,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:51,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:51,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:51,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:51,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741838_1014 (size=12001) 2024-11-28T06:53:51,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T06:53:51,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:51,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:53:51,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:51,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:51,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:51,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:52,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:52,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36870 deadline: 1732776892017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:52,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:52,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:52,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:52,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:52,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776892017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:52,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:52,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:52,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:52,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:52,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776892037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:52,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:52,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:52,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:52,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:52,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776892046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:52,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:52,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:52,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:52,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:52,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776892047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:52,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:52,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:52,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:52,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T06:53:52,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:52,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:53:52,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:52,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:52,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:52,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:52,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T06:53:52,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T06:53:52,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:52,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:53:52,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:52,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:52,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:52,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:52,273 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/99de9edc2c644e52a35fb298627b045e 2024-11-28T06:53:52,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/6fc5d1b2bbed49fb85c747f961200581 is 50, key is test_row_0/C:col10/1732776831486/Put/seqid=0 2024-11-28T06:53:52,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36870 deadline: 1732776892326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:52,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:52,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:52,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:52,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:52,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776892336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:52,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:52,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:52,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:52,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776892358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:52,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:52,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:52,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:52,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776892358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:52,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:52,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:52,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:52,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776892366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:52,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:52,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:52,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:52,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T06:53:52,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:52,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:53:52,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:52,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:52,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:52,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:52,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741839_1015 (size=12001) 2024-11-28T06:53:52,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T06:53:52,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:52,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:53:52,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:52,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:52,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T06:53:52,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T06:53:52,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:52,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:53:52,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:52,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:52,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:52,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:52,807 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/6fc5d1b2bbed49fb85c747f961200581 2024-11-28T06:53:52,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/40db68802f584dd8afa8de305e7cd21b as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/40db68802f584dd8afa8de305e7cd21b 2024-11-28T06:53:52,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/40db68802f584dd8afa8de305e7cd21b, entries=200, sequenceid=13, filesize=14.0 K 2024-11-28T06:53:52,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/99de9edc2c644e52a35fb298627b045e as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/99de9edc2c644e52a35fb298627b045e 2024-11-28T06:53:52,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T06:53:52,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:52,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:53:52,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:52,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:52,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:52,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:52,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/99de9edc2c644e52a35fb298627b045e, entries=150, sequenceid=13, filesize=11.7 K 2024-11-28T06:53:52,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/6fc5d1b2bbed49fb85c747f961200581 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/6fc5d1b2bbed49fb85c747f961200581 2024-11-28T06:53:52,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:52,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/6fc5d1b2bbed49fb85c747f961200581, entries=150, sequenceid=13, filesize=11.7 K 2024-11-28T06:53:52,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36870 deadline: 1732776892848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:52,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:52,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:52,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:52,908 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for d6ca3b1ced864db1c50bb8f904a838f3 in 1409ms, sequenceid=13, compaction requested=false 2024-11-28T06:53:52,910 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-28T06:53:52,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:53:52,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:53:52,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6ca3b1ced864db1c50bb8f904a838f3 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-28T06:53:52,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732776831607, max timestamp=1732776832902Num uniques 859; 2024-11-28T06:53:52,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T06:53:52,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732776831607, max timestamp=1732776832909Num uniques 859; 2024-11-28T06:53:52,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T06:53:52,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=A 2024-11-28T06:53:52,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:53:52,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=B 2024-11-28T06:53:52,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:53:52,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=C 2024-11-28T06:53:52,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:53:52,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/b50a969913594b0ea4c7126072943c78 is 50, key is test_row_0/A:col10/1732776831607/Put/seqid=0 2024-11-28T06:53:53,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741840_1016 (size=16681) 2024-11-28T06:53:53,008 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/b50a969913594b0ea4c7126072943c78 2024-11-28T06:53:53,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/9fac955e110c460ab5ae3168d460fe89 is 50, key is test_row_0/B:col10/1732776831607/Put/seqid=0 2024-11-28T06:53:53,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T06:53:53,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:53,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:53:53,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:53,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:53,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:53,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:53,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741841_1017 (size=12001) 2024-11-28T06:53:53,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:53,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776893153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:53,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:53,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:53,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:53,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:53,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776893180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:53,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:53,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:53,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:53,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:53,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776893183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:53,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:53,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:53,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:53,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:53,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776893188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:53,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:53,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:53,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:53,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T06:53:53,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:53,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:53:53,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:53,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:53,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:53,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:53,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:53,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776893297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:53,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:53,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:53,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:53,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:53,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776893307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:53,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:53,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:53,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:53,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:53,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:53,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776893308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:53,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776893309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:53,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:53,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:53,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:53,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:53,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T06:53:53,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:53,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:53:53,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:53,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:53,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:53,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:53,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/9fac955e110c460ab5ae3168d460fe89 2024-11-28T06:53:53,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:53,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776893516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:53,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:53,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:53,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:53,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:53,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776893526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:53,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:53,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:53,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:53,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/7dc57186d65740a885469b06a0fa4d1e is 50, key is test_row_0/C:col10/1732776831607/Put/seqid=0 2024-11-28T06:53:53,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:53,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776893536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:53,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:53,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:53,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776893537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:53,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:53,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:53,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:53,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:53,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:53,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T06:53:53,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:53,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:53:53,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:53,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:53,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:53,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:53,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741842_1018 (size=12001) 2024-11-28T06:53:53,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T06:53:53,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T06:53:53,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:53,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:53:53,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:53,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:53,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:53,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:53,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:53,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776893827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:53,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:53,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:53,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:53,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:53,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776893836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:53,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:53,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:53,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:53,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:53,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:53,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776893847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:53,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776893847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:53,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:53,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:53,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:53,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:53,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:53,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:53,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T06:53:53,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:53,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:53:53,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:53,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:53,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:53,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:53,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:53,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36870 deadline: 1732776893916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:53,924 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_0' in d6ca3b1ced864db1c50bb8f904a838f3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2289 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:53:53,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:53,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:53,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:53,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/7dc57186d65740a885469b06a0fa4d1e 2024-11-28T06:53:53,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/b50a969913594b0ea4c7126072943c78 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/b50a969913594b0ea4c7126072943c78 2024-11-28T06:53:54,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T06:53:54,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:54,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:53:54,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:54,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:54,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:54,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/b50a969913594b0ea4c7126072943c78, entries=250, sequenceid=36, filesize=16.3 K 2024-11-28T06:53:54,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:54,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/9fac955e110c460ab5ae3168d460fe89 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/9fac955e110c460ab5ae3168d460fe89 2024-11-28T06:53:54,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/9fac955e110c460ab5ae3168d460fe89, entries=150, sequenceid=36, filesize=11.7 K 2024-11-28T06:53:54,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/7dc57186d65740a885469b06a0fa4d1e as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/7dc57186d65740a885469b06a0fa4d1e 2024-11-28T06:53:54,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/7dc57186d65740a885469b06a0fa4d1e, entries=150, sequenceid=36, filesize=11.7 K 2024-11-28T06:53:54,061 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~71.56 KB/73280, heapSize ~203.97 KB/208864, currentSize=107.34 KB/109920 for d6ca3b1ced864db1c50bb8f904a838f3 in 1131ms, sequenceid=36, compaction requested=false 2024-11-28T06:53:54,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:53:54,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T06:53:54,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:54,167 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing d6ca3b1ced864db1c50bb8f904a838f3 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T06:53:54,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=A 2024-11-28T06:53:54,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:53:54,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=B 2024-11-28T06:53:54,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:53:54,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=C 2024-11-28T06:53:54,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:53:54,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/2cd3cb344ce443059d511a94e031308a is 50, key is test_row_0/A:col10/1732776833146/Put/seqid=0 2024-11-28T06:53:54,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741843_1019 (size=12001) 2024-11-28T06:53:54,200 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/2cd3cb344ce443059d511a94e031308a 2024-11-28T06:53:54,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/eff14829789e4911848f48dd092bd51f is 50, key is test_row_0/B:col10/1732776833146/Put/seqid=0 2024-11-28T06:53:54,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741844_1020 (size=12001) 2024-11-28T06:53:54,248 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/eff14829789e4911848f48dd092bd51f 2024-11-28T06:53:54,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/8f815f8f4d2d45b7bbd88e4442ecaa04 is 50, key is test_row_0/C:col10/1732776833146/Put/seqid=0 2024-11-28T06:53:54,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741845_1021 (size=12001) 2024-11-28T06:53:54,335 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/8f815f8f4d2d45b7bbd88e4442ecaa04 2024-11-28T06:53:54,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:53:54,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:53:54,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/2cd3cb344ce443059d511a94e031308a as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/2cd3cb344ce443059d511a94e031308a 2024-11-28T06:53:54,368 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/2cd3cb344ce443059d511a94e031308a, entries=150, sequenceid=55, filesize=11.7 K 2024-11-28T06:53:54,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/eff14829789e4911848f48dd092bd51f as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/eff14829789e4911848f48dd092bd51f 2024-11-28T06:53:54,391 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/eff14829789e4911848f48dd092bd51f, entries=150, sequenceid=55, filesize=11.7 K 2024-11-28T06:53:54,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:54,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776894383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:54,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/8f815f8f4d2d45b7bbd88e4442ecaa04 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/8f815f8f4d2d45b7bbd88e4442ecaa04 2024-11-28T06:53:54,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:54,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:54,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:54,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:54,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776894387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:54,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:54,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:54,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:54,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776894392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:54,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:54,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:54,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:54,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:54,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776894395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:54,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:54,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:54,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:54,418 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/8f815f8f4d2d45b7bbd88e4442ecaa04, entries=150, sequenceid=55, filesize=11.7 K 2024-11-28T06:53:54,421 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for d6ca3b1ced864db1c50bb8f904a838f3 in 253ms, sequenceid=55, compaction requested=true 2024-11-28T06:53:54,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:53:54,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:54,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-28T06:53:54,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-28T06:53:54,428 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-28T06:53:54,428 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9570 sec 2024-11-28T06:53:54,434 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.9760 sec 2024-11-28T06:53:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:53:54,511 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6ca3b1ced864db1c50bb8f904a838f3 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T06:53:54,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=A 2024-11-28T06:53:54,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:53:54,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=B 2024-11-28T06:53:54,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:53:54,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=C 2024-11-28T06:53:54,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:53:54,529 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/2399290eca8d4f2ea87435b1e9f7920e is 50, key is test_row_0/A:col10/1732776834387/Put/seqid=0 2024-11-28T06:53:54,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741846_1022 (size=14341) 2024-11-28T06:53:54,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/2399290eca8d4f2ea87435b1e9f7920e 2024-11-28T06:53:54,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:54,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776894569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:54,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:54,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:54,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:54,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776894584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:54,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:54,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:54,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:54,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:54,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776894587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:54,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:54,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:54,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776894587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:54,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:54,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:54,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:54,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:54,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:54,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/1396152e7dd941d9bda378b13b1c12ca is 50, key is test_row_0/B:col10/1732776834387/Put/seqid=0 2024-11-28T06:53:54,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741847_1023 (size=12001) 2024-11-28T06:53:54,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:54,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776894696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:54,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:54,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:54,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:54,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:54,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776894696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:54,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:54,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:54,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:54,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776894706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:54,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:54,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:54,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:54,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:54,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776894706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:54,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:54,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:54,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:54,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:54,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776894906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:54,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:54,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:54,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:54,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776894916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:54,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:54,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:54,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:54,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:54,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776894926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:54,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776894926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:54,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:54,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:54,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:54,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:54,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:54,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:55,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/1396152e7dd941d9bda378b13b1c12ca 2024-11-28T06:53:55,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/8950d83064cd414c8748fab27cca64fd is 50, key is test_row_0/C:col10/1732776834387/Put/seqid=0 2024-11-28T06:53:55,095 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T06:53:55,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741848_1024 (size=12001) 2024-11-28T06:53:55,209 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-28T06:53:55,211 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-28T06:53:55,212 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-28T06:53:55,214 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-28T06:53:55,214 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-28T06:53:55,215 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-28T06:53:55,215 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-28T06:53:55,216 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-28T06:53:55,216 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-28T06:53:55,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:55,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776895226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:55,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:55,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:55,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:55,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:55,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776895236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:55,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:55,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:55,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:55,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:55,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776895246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:55,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:55,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:55,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:55,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:55,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776895256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:55,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:55,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:55,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:55,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/8950d83064cd414c8748fab27cca64fd 2024-11-28T06:53:55,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/2399290eca8d4f2ea87435b1e9f7920e as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/2399290eca8d4f2ea87435b1e9f7920e 2024-11-28T06:53:55,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/2399290eca8d4f2ea87435b1e9f7920e, entries=200, sequenceid=69, filesize=14.0 K 2024-11-28T06:53:55,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/1396152e7dd941d9bda378b13b1c12ca as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/1396152e7dd941d9bda378b13b1c12ca 2024-11-28T06:53:55,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/1396152e7dd941d9bda378b13b1c12ca, entries=150, sequenceid=69, filesize=11.7 K 2024-11-28T06:53:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T06:53:55,636 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:53:55,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/8950d83064cd414c8748fab27cca64fd as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/8950d83064cd414c8748fab27cca64fd 2024-11-28T06:53:55,637 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:53:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-28T06:53:55,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-28T06:53:55,641 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:53:55,643 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:53:55,643 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:53:55,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/8950d83064cd414c8748fab27cca64fd, entries=150, sequenceid=69, filesize=11.7 K 2024-11-28T06:53:55,651 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for d6ca3b1ced864db1c50bb8f904a838f3 in 1141ms, sequenceid=69, compaction requested=true 2024-11-28T06:53:55,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:53:55,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ca3b1ced864db1c50bb8f904a838f3:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:53:55,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:53:55,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ca3b1ced864db1c50bb8f904a838f3:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:53:55,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:53:55,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ca3b1ced864db1c50bb8f904a838f3:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:53:55,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:53:55,668 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T06:53:55,673 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T06:53:55,674 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T06:53:55,679 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57364 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T06:53:55,679 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): d6ca3b1ced864db1c50bb8f904a838f3/B is initiating minor compaction (all files) 2024-11-28T06:53:55,679 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): d6ca3b1ced864db1c50bb8f904a838f3/A is initiating minor compaction (all files) 2024-11-28T06:53:55,679 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ca3b1ced864db1c50bb8f904a838f3/A in TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:55,680 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/40db68802f584dd8afa8de305e7cd21b, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/b50a969913594b0ea4c7126072943c78, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/2cd3cb344ce443059d511a94e031308a, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/2399290eca8d4f2ea87435b1e9f7920e] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp, totalSize=56.0 K 2024-11-28T06:53:55,682 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40db68802f584dd8afa8de305e7cd21b, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732776831399 2024-11-28T06:53:55,683 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting b50a969913594b0ea4c7126072943c78, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732776831607 2024-11-28T06:53:55,684 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cd3cb344ce443059d511a94e031308a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732776833146 2024-11-28T06:53:55,686 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2399290eca8d4f2ea87435b1e9f7920e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732776834376 2024-11-28T06:53:55,690 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ca3b1ced864db1c50bb8f904a838f3/B in TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:55,691 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/99de9edc2c644e52a35fb298627b045e, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/9fac955e110c460ab5ae3168d460fe89, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/eff14829789e4911848f48dd092bd51f, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/1396152e7dd941d9bda378b13b1c12ca] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp, totalSize=46.9 K 2024-11-28T06:53:55,692 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 99de9edc2c644e52a35fb298627b045e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732776831399 2024-11-28T06:53:55,693 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fac955e110c460ab5ae3168d460fe89, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732776831607 2024-11-28T06:53:55,695 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting eff14829789e4911848f48dd092bd51f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732776833146 2024-11-28T06:53:55,696 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 1396152e7dd941d9bda378b13b1c12ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732776834376 2024-11-28T06:53:55,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-28T06:53:55,751 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ca3b1ced864db1c50bb8f904a838f3#A#compaction#12 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:53:55,752 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/a4c78a863a4f466d9ad433f3959f4f89 is 50, key is test_row_0/A:col10/1732776834387/Put/seqid=0 2024-11-28T06:53:55,758 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ca3b1ced864db1c50bb8f904a838f3#B#compaction#13 average throughput is 0.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:53:55,758 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/f58dd164cd1e46278e7f234268740843 is 50, key is test_row_0/B:col10/1732776834387/Put/seqid=0 2024-11-28T06:53:55,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:53:55,776 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6ca3b1ced864db1c50bb8f904a838f3 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T06:53:55,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741849_1025 (size=12139) 2024-11-28T06:53:55,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-28T06:53:55,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:55,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:53:55,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:55,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:55,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:55,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732776834560, max timestamp=1732776835776Num uniques 859; 2024-11-28T06:53:55,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T06:53:55,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741850_1026 (size=12139) 2024-11-28T06:53:55,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=A 2024-11-28T06:53:55,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:53:55,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=B 2024-11-28T06:53:55,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:53:55,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=C 2024-11-28T06:53:55,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:53:55,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/3d1731be43584894be2ca036799be8d2 is 50, key is test_row_0/A:col10/1732776835774/Put/seqid=0 2024-11-28T06:53:55,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741851_1027 (size=12001) 2024-11-28T06:53:55,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/3d1731be43584894be2ca036799be8d2 2024-11-28T06:53:55,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/d2ae48590ba34986b7874734538f18a4 is 50, key is test_row_0/B:col10/1732776835774/Put/seqid=0 2024-11-28T06:53:55,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741852_1028 (size=12001) 2024-11-28T06:53:55,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/d2ae48590ba34986b7874734538f18a4 2024-11-28T06:53:55,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776895893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:55,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:55,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:55,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:55,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776895901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:55,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:55,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:55,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:55,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/3fd11127f7614d42a5113894fdb0d78d is 50, key is test_row_0/C:col10/1732776835774/Put/seqid=0 2024-11-28T06:53:55,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776895925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:55,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:55,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:55,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:55,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776895928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:55,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:55,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:55,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:55,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741853_1029 (size=12001) 2024-11-28T06:53:55,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-28T06:53:55,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:55,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:53:55,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:55,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:55,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:55,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36870 deadline: 1732776895935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-28T06:53:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:55,962 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_0' in d6ca3b1ced864db1c50bb8f904a838f3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4332 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:53:55,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:55,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:55,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:55,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/3fd11127f7614d42a5113894fdb0d78d 2024-11-28T06:53:55,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/3d1731be43584894be2ca036799be8d2 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/3d1731be43584894be2ca036799be8d2 2024-11-28T06:53:55,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/3d1731be43584894be2ca036799be8d2, entries=150, sequenceid=90, filesize=11.7 K 2024-11-28T06:53:55,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/d2ae48590ba34986b7874734538f18a4 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/d2ae48590ba34986b7874734538f18a4 2024-11-28T06:53:56,016 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/d2ae48590ba34986b7874734538f18a4, entries=150, sequenceid=90, filesize=11.7 K 2024-11-28T06:53:56,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/3fd11127f7614d42a5113894fdb0d78d as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/3fd11127f7614d42a5113894fdb0d78d 2024-11-28T06:53:56,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776896036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:56,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:56,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:56,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:56,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:56,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776896036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:56,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:56,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:56,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:56,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/3fd11127f7614d42a5113894fdb0d78d, entries=150, sequenceid=90, filesize=11.7 K 2024-11-28T06:53:56,044 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~89.45 KB/91600, heapSize ~262.06 KB/268352, currentSize=93.93 KB/96180 for d6ca3b1ced864db1c50bb8f904a838f3 in 268ms, sequenceid=90, compaction requested=true 2024-11-28T06:53:56,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:53:56,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ca3b1ced864db1c50bb8f904a838f3:A, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:53:56,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-28T06:53:56,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ca3b1ced864db1c50bb8f904a838f3:B, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:53:56,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-28T06:53:56,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ca3b1ced864db1c50bb8f904a838f3:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:53:56,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-28T06:53:56,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:53:56,063 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6ca3b1ced864db1c50bb8f904a838f3 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T06:53:56,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=A 2024-11-28T06:53:56,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:53:56,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=B 2024-11-28T06:53:56,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:53:56,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=C 2024-11-28T06:53:56,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:53:56,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/07dd5a78d1c34702b36ba234cd327f4e is 50, key is test_row_0/A:col10/1732776836060/Put/seqid=0 2024-11-28T06:53:56,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741854_1030 (size=12001) 2024-11-28T06:53:56,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/07dd5a78d1c34702b36ba234cd327f4e 2024-11-28T06:53:56,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-28T06:53:56,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:56,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:53:56,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:56,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:56,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:56,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:56,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/c9cf2ddd2728415e9555f60b1cc68c0a is 50, key is test_row_0/B:col10/1732776836060/Put/seqid=0 2024-11-28T06:53:56,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741855_1031 (size=12001) 2024-11-28T06:53:56,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:56,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776896149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:56,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:56,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:56,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:56,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:56,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776896150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:56,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/c9cf2ddd2728415e9555f60b1cc68c0a 2024-11-28T06:53:56,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:56,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:56,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:56,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/87bbb20d080c48498a0e5dea3eb7bdf6 is 50, key is test_row_0/C:col10/1732776836060/Put/seqid=0 2024-11-28T06:53:56,224 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/a4c78a863a4f466d9ad433f3959f4f89 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/a4c78a863a4f466d9ad433f3959f4f89 2024-11-28T06:53:56,230 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/f58dd164cd1e46278e7f234268740843 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/f58dd164cd1e46278e7f234268740843 2024-11-28T06:53:56,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741856_1032 (size=12001) 2024-11-28T06:53:56,249 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d6ca3b1ced864db1c50bb8f904a838f3/A of d6ca3b1ced864db1c50bb8f904a838f3 into a4c78a863a4f466d9ad433f3959f4f89(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:53:56,249 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:53:56,253 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., storeName=d6ca3b1ced864db1c50bb8f904a838f3/A, priority=12, startTime=1732776835653; duration=0sec 2024-11-28T06:53:56,253 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-28T06:53:56,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:56,254 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d6ca3b1ced864db1c50bb8f904a838f3/B of d6ca3b1ced864db1c50bb8f904a838f3 into f58dd164cd1e46278e7f234268740843(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:53:56,254 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ca3b1ced864db1c50bb8f904a838f3:A 2024-11-28T06:53:56,254 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:53:56,254 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., storeName=d6ca3b1ced864db1c50bb8f904a838f3/B, priority=12, startTime=1732776835665; duration=0sec 2024-11-28T06:53:56,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776896246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:56,254 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ca3b1ced864db1c50bb8f904a838f3:A 2024-11-28T06:53:56,254 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-28T06:53:56,254 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-28T06:53:56,254 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ca3b1ced864db1c50bb8f904a838f3:B 2024-11-28T06:53:56,254 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ca3b1ced864db1c50bb8f904a838f3:B 2024-11-28T06:53:56,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:56,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:56,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:56,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:56,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776896247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:56,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:56,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:56,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:56,258 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-28T06:53:56,258 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): d6ca3b1ced864db1c50bb8f904a838f3/C is initiating minor compaction (all files) 2024-11-28T06:53:56,258 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 5 compacting, 0 eligible, 16 blocking 2024-11-28T06:53:56,258 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-28T06:53:56,258 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ca3b1ced864db1c50bb8f904a838f3/C in TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:56,259 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-28T06:53:56,259 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. because compaction request was cancelled 2024-11-28T06:53:56,259 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ca3b1ced864db1c50bb8f904a838f3:C 2024-11-28T06:53:56,259 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-28T06:53:56,261 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-28T06:53:56,261 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-28T06:53:56,261 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. because compaction request was cancelled 2024-11-28T06:53:56,261 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ca3b1ced864db1c50bb8f904a838f3:B 2024-11-28T06:53:56,262 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-28T06:53:56,262 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/6fc5d1b2bbed49fb85c747f961200581, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/7dc57186d65740a885469b06a0fa4d1e, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/8f815f8f4d2d45b7bbd88e4442ecaa04, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/8950d83064cd414c8748fab27cca64fd, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/3fd11127f7614d42a5113894fdb0d78d] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp, totalSize=58.6 K 2024-11-28T06:53:56,263 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-28T06:53:56,264 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-28T06:53:56,264 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. because compaction request was cancelled 2024-11-28T06:53:56,264 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ca3b1ced864db1c50bb8f904a838f3:A 2024-11-28T06:53:56,264 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fc5d1b2bbed49fb85c747f961200581, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732776831399 2024-11-28T06:53:56,267 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7dc57186d65740a885469b06a0fa4d1e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732776831607 2024-11-28T06:53:56,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-28T06:53:56,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:56,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776896267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:56,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:56,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:56,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:56,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:56,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776896267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:56,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:56,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:56,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:56,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-28T06:53:56,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:56,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:53:56,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:56,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:56,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:56,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:56,281 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f815f8f4d2d45b7bbd88e4442ecaa04, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732776833146 2024-11-28T06:53:56,282 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8950d83064cd414c8748fab27cca64fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732776834376 2024-11-28T06:53:56,283 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fd11127f7614d42a5113894fdb0d78d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732776834582 2024-11-28T06:53:56,331 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ca3b1ced864db1c50bb8f904a838f3#C#compaction#20 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:53:56,332 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/5751bb49721c41288cd46ed0d62623dc is 50, key is test_row_0/C:col10/1732776835774/Put/seqid=0 2024-11-28T06:53:56,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741857_1033 (size=12173) 2024-11-28T06:53:56,373 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/5751bb49721c41288cd46ed0d62623dc as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/5751bb49721c41288cd46ed0d62623dc 2024-11-28T06:53:56,388 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in d6ca3b1ced864db1c50bb8f904a838f3/C of d6ca3b1ced864db1c50bb8f904a838f3 into 5751bb49721c41288cd46ed0d62623dc(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:53:56,388 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:53:56,388 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., storeName=d6ca3b1ced864db1c50bb8f904a838f3/C, priority=11, startTime=1732776836045; duration=0sec 2024-11-28T06:53:56,389 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:53:56,389 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ca3b1ced864db1c50bb8f904a838f3:C 2024-11-28T06:53:56,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-28T06:53:56,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:56,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:53:56,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:56,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:56,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:56,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:56,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:56,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776896476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:56,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:56,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:56,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:56,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:56,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776896486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:56,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:56,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:56,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:56,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:56,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776896567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:56,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:56,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:56,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:56,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:56,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776896567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:56,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:56,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:56,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:56,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-28T06:53:56,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:56,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:53:56,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:56,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:56,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:56,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:56,636 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/87bbb20d080c48498a0e5dea3eb7bdf6 2024-11-28T06:53:56,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/07dd5a78d1c34702b36ba234cd327f4e as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/07dd5a78d1c34702b36ba234cd327f4e 2024-11-28T06:53:56,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/07dd5a78d1c34702b36ba234cd327f4e, entries=150, sequenceid=109, filesize=11.7 K 2024-11-28T06:53:56,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/c9cf2ddd2728415e9555f60b1cc68c0a as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/c9cf2ddd2728415e9555f60b1cc68c0a 2024-11-28T06:53:56,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/c9cf2ddd2728415e9555f60b1cc68c0a, entries=150, sequenceid=109, filesize=11.7 K 2024-11-28T06:53:56,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/87bbb20d080c48498a0e5dea3eb7bdf6 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/87bbb20d080c48498a0e5dea3eb7bdf6 2024-11-28T06:53:56,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/87bbb20d080c48498a0e5dea3eb7bdf6, entries=150, sequenceid=109, filesize=11.7 K 2024-11-28T06:53:56,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for d6ca3b1ced864db1c50bb8f904a838f3 in 677ms, sequenceid=109, compaction requested=true 2024-11-28T06:53:56,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:53:56,741 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:53:56,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ca3b1ced864db1c50bb8f904a838f3:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:53:56,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:53:56,742 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:53:56,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ca3b1ced864db1c50bb8f904a838f3:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:53:56,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:53:56,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ca3b1ced864db1c50bb8f904a838f3:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:53:56,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:53:56,745 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:53:56,745 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): d6ca3b1ced864db1c50bb8f904a838f3/A is initiating minor compaction (all files) 2024-11-28T06:53:56,745 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ca3b1ced864db1c50bb8f904a838f3/A in TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:56,746 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/a4c78a863a4f466d9ad433f3959f4f89, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/3d1731be43584894be2ca036799be8d2, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/07dd5a78d1c34702b36ba234cd327f4e] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp, totalSize=35.3 K 2024-11-28T06:53:56,746 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:53:56,746 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): d6ca3b1ced864db1c50bb8f904a838f3/B is initiating minor compaction (all files) 2024-11-28T06:53:56,746 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ca3b1ced864db1c50bb8f904a838f3/B in TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:56,747 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/f58dd164cd1e46278e7f234268740843, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/d2ae48590ba34986b7874734538f18a4, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/c9cf2ddd2728415e9555f60b1cc68c0a] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp, totalSize=35.3 K 2024-11-28T06:53:56,747 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4c78a863a4f466d9ad433f3959f4f89, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732776834376 2024-11-28T06:53:56,748 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting f58dd164cd1e46278e7f234268740843, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732776834376 2024-11-28T06:53:56,749 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d1731be43584894be2ca036799be8d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732776834582 2024-11-28T06:53:56,750 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting d2ae48590ba34986b7874734538f18a4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732776834582 2024-11-28T06:53:56,751 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07dd5a78d1c34702b36ba234cd327f4e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732776835893 2024-11-28T06:53:56,752 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting c9cf2ddd2728415e9555f60b1cc68c0a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732776835893 2024-11-28T06:53:56,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-28T06:53:56,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:56,767 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing d6ca3b1ced864db1c50bb8f904a838f3 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T06:53:56,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=A 2024-11-28T06:53:56,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:53:56,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=B 2024-11-28T06:53:56,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:53:56,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=C 2024-11-28T06:53:56,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:53:56,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-28T06:53:56,776 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ca3b1ced864db1c50bb8f904a838f3#B#compaction#21 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:53:56,776 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ca3b1ced864db1c50bb8f904a838f3#A#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:53:56,776 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/c91ab63f9f274667bf242ee4864b0ac9 is 50, key is test_row_0/B:col10/1732776836060/Put/seqid=0 2024-11-28T06:53:56,777 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/6d3dcc6500184144a0b5c37d02e6a002 is 50, key is test_row_0/A:col10/1732776836060/Put/seqid=0 2024-11-28T06:53:56,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/9c6ca5993ec84d8ba241733fdd5c2153 is 50, key is test_row_0/A:col10/1732776836102/Put/seqid=0 2024-11-28T06:53:56,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:53:56,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:53:56,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741859_1035 (size=12241) 2024-11-28T06:53:56,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741860_1036 (size=12001) 2024-11-28T06:53:56,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741858_1034 (size=12241) 2024-11-28T06:53:56,823 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/9c6ca5993ec84d8ba241733fdd5c2153 2024-11-28T06:53:56,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/9d5e7da751bd4aeb9935e197f3d68a3b is 50, key is test_row_0/B:col10/1732776836102/Put/seqid=0 2024-11-28T06:53:56,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:56,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776896861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:56,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:56,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:56,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:56,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:56,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776896863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:56,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:56,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:56,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:56,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741861_1037 (size=12001) 2024-11-28T06:53:56,910 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/9d5e7da751bd4aeb9935e197f3d68a3b 2024-11-28T06:53:56,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/c5be36c79319478db2d897a12d91904f is 50, key is test_row_0/C:col10/1732776836102/Put/seqid=0 2024-11-28T06:53:56,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741862_1038 (size=12001) 2024-11-28T06:53:56,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:56,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776896976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:56,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:56,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:56,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:56,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:56,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776896976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:56,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:56,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:56,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:57,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776897075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:57,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:57,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:57,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:57,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776897075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:57,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:57,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:57,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:57,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776897186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:57,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:57,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:57,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:57,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776897197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:57,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:57,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:57,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:57,225 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/6d3dcc6500184144a0b5c37d02e6a002 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/6d3dcc6500184144a0b5c37d02e6a002 2024-11-28T06:53:57,241 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/c91ab63f9f274667bf242ee4864b0ac9 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/c91ab63f9f274667bf242ee4864b0ac9 2024-11-28T06:53:57,241 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ca3b1ced864db1c50bb8f904a838f3/A of d6ca3b1ced864db1c50bb8f904a838f3 into 6d3dcc6500184144a0b5c37d02e6a002(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:53:57,241 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:53:57,242 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., storeName=d6ca3b1ced864db1c50bb8f904a838f3/A, priority=13, startTime=1732776836741; duration=0sec 2024-11-28T06:53:57,242 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:53:57,242 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ca3b1ced864db1c50bb8f904a838f3:A 2024-11-28T06:53:57,242 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-28T06:53:57,245 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-28T06:53:57,245 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-28T06:53:57,245 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. because compaction request was cancelled 2024-11-28T06:53:57,245 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ca3b1ced864db1c50bb8f904a838f3:C 2024-11-28T06:53:57,254 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ca3b1ced864db1c50bb8f904a838f3/B of d6ca3b1ced864db1c50bb8f904a838f3 into c91ab63f9f274667bf242ee4864b0ac9(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:53:57,254 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:53:57,254 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., storeName=d6ca3b1ced864db1c50bb8f904a838f3/B, priority=13, startTime=1732776836742; duration=0sec 2024-11-28T06:53:57,255 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:53:57,255 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ca3b1ced864db1c50bb8f904a838f3:B 2024-11-28T06:53:57,379 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/c5be36c79319478db2d897a12d91904f 2024-11-28T06:53:57,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/9c6ca5993ec84d8ba241733fdd5c2153 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/9c6ca5993ec84d8ba241733fdd5c2153 2024-11-28T06:53:57,404 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/9c6ca5993ec84d8ba241733fdd5c2153, entries=150, sequenceid=125, filesize=11.7 K 2024-11-28T06:53:57,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/9d5e7da751bd4aeb9935e197f3d68a3b as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/9d5e7da751bd4aeb9935e197f3d68a3b 2024-11-28T06:53:57,433 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/9d5e7da751bd4aeb9935e197f3d68a3b, entries=150, sequenceid=125, filesize=11.7 K 2024-11-28T06:53:57,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/c5be36c79319478db2d897a12d91904f as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/c5be36c79319478db2d897a12d91904f 2024-11-28T06:53:57,453 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/c5be36c79319478db2d897a12d91904f, entries=150, sequenceid=125, filesize=11.7 K 2024-11-28T06:53:57,455 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for d6ca3b1ced864db1c50bb8f904a838f3 in 688ms, sequenceid=125, compaction requested=true 2024-11-28T06:53:57,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:53:57,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:57,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-28T06:53:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-28T06:53:57,464 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-28T06:53:57,464 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8150 sec 2024-11-28T06:53:57,468 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.8290 sec 2024-11-28T06:53:57,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:53:57,504 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6ca3b1ced864db1c50bb8f904a838f3 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T06:53:57,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=A 2024-11-28T06:53:57,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:53:57,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=B 2024-11-28T06:53:57,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:53:57,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=C 2024-11-28T06:53:57,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:53:57,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/6a0e2737f1cc453a984495dd8199368b is 50, key is test_row_0/A:col10/1732776836859/Put/seqid=0 2024-11-28T06:53:57,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741863_1039 (size=14541) 2024-11-28T06:53:57,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/6a0e2737f1cc453a984495dd8199368b 2024-11-28T06:53:57,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:57,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776897538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:57,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:57,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:57,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:57,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:57,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776897546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:57,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:57,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:57,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:57,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/7fe75095178b4ed5b25fd13b367e60b6 is 50, key is test_row_0/B:col10/1732776836859/Put/seqid=0 2024-11-28T06:53:57,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741864_1040 (size=12151) 2024-11-28T06:53:57,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:57,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:57,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776897656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:57,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:57,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:57,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:57,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776897655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:57,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:57,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:57,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:57,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-28T06:53:57,786 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:53:57,787 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:53:57,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-28T06:53:57,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-28T06:53:57,791 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:53:57,793 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:53:57,793 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:53:57,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:57,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776897865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:57,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:57,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:57,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:57,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:57,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776897865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:57,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:57,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:57,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:57,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-28T06:53:57,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T06:53:57,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:57,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:53:57,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:57,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:57,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:57,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:57,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/7fe75095178b4ed5b25fd13b367e60b6 2024-11-28T06:53:58,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/4b608daed9b24b46897dafb85f89087a is 50, key is test_row_0/C:col10/1732776836859/Put/seqid=0 2024-11-28T06:53:58,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741865_1041 (size=12151) 2024-11-28T06:53:58,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/4b608daed9b24b46897dafb85f89087a 2024-11-28T06:53:58,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/6a0e2737f1cc453a984495dd8199368b as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/6a0e2737f1cc453a984495dd8199368b 2024-11-28T06:53:58,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/6a0e2737f1cc453a984495dd8199368b, entries=200, sequenceid=147, filesize=14.2 K 2024-11-28T06:53:58,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/7fe75095178b4ed5b25fd13b367e60b6 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/7fe75095178b4ed5b25fd13b367e60b6 2024-11-28T06:53:58,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/7fe75095178b4ed5b25fd13b367e60b6, entries=150, sequenceid=147, filesize=11.9 K 2024-11-28T06:53:58,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/4b608daed9b24b46897dafb85f89087a as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/4b608daed9b24b46897dafb85f89087a 2024-11-28T06:53:58,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/4b608daed9b24b46897dafb85f89087a, entries=150, sequenceid=147, filesize=11.9 K 2024-11-28T06:53:58,085 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for d6ca3b1ced864db1c50bb8f904a838f3 in 582ms, sequenceid=147, compaction requested=true 2024-11-28T06:53:58,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:53:58,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ca3b1ced864db1c50bb8f904a838f3:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:53:58,086 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:53:58,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:53:58,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ca3b1ced864db1c50bb8f904a838f3:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:53:58,086 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:53:58,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:53:58,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ca3b1ced864db1c50bb8f904a838f3:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:53:58,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:53:58,088 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38783 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:53:58,088 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): d6ca3b1ced864db1c50bb8f904a838f3/A is initiating minor compaction (all files) 2024-11-28T06:53:58,088 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ca3b1ced864db1c50bb8f904a838f3/A in TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:58,089 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/6d3dcc6500184144a0b5c37d02e6a002, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/9c6ca5993ec84d8ba241733fdd5c2153, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/6a0e2737f1cc453a984495dd8199368b] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp, totalSize=37.9 K 2024-11-28T06:53:58,089 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:53:58,089 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): d6ca3b1ced864db1c50bb8f904a838f3/B is initiating minor compaction (all files) 2024-11-28T06:53:58,089 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ca3b1ced864db1c50bb8f904a838f3/B in TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:58,089 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/c91ab63f9f274667bf242ee4864b0ac9, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/9d5e7da751bd4aeb9935e197f3d68a3b, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/7fe75095178b4ed5b25fd13b367e60b6] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp, totalSize=35.5 K 2024-11-28T06:53:58,090 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d3dcc6500184144a0b5c37d02e6a002, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732776835893 2024-11-28T06:53:58,090 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting c91ab63f9f274667bf242ee4864b0ac9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732776835893 2024-11-28T06:53:58,091 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d5e7da751bd4aeb9935e197f3d68a3b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1732776836102 2024-11-28T06:53:58,091 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c6ca5993ec84d8ba241733fdd5c2153, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1732776836102 2024-11-28T06:53:58,092 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fe75095178b4ed5b25fd13b367e60b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732776836848 2024-11-28T06:53:58,093 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a0e2737f1cc453a984495dd8199368b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732776836848 2024-11-28T06:53:58,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:53:58,100 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6ca3b1ced864db1c50bb8f904a838f3 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T06:53:58,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=A 2024-11-28T06:53:58,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:53:58,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=B 2024-11-28T06:53:58,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:53:58,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=C 2024-11-28T06:53:58,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T06:53:58,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:58,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:53:58,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:53:58,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:58,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:58,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:58,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:58,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-28T06:53:58,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/dd28d6c98e0d4e9286d6b71539737d42 is 50, key is test_row_0/A:col10/1732776838097/Put/seqid=0 2024-11-28T06:53:58,119 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ca3b1ced864db1c50bb8f904a838f3#B#compaction#30 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:53:58,120 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/b9d122b9ef0b4f6687361e34760d1d91 is 50, key is test_row_0/B:col10/1732776836859/Put/seqid=0 2024-11-28T06:53:58,131 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ca3b1ced864db1c50bb8f904a838f3#A#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:53:58,132 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/a9e13f7e074441608fa9c7f6f7e00529 is 50, key is test_row_0/A:col10/1732776836859/Put/seqid=0 2024-11-28T06:53:58,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741866_1042 (size=12493) 2024-11-28T06:53:58,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:58,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776898185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:58,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:58,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:58,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:58,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:58,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776898191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:58,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:58,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776898191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:58,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:58,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:58,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:58,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:58,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776898194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:58,208 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/b9d122b9ef0b4f6687361e34760d1d91 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/b9d122b9ef0b4f6687361e34760d1d91 2024-11-28T06:53:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:58,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:58,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:58,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741868_1044 (size=12493) 2024-11-28T06:53:58,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741867_1043 (size=14541) 2024-11-28T06:53:58,220 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/dd28d6c98e0d4e9286d6b71539737d42 2024-11-28T06:53:58,240 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ca3b1ced864db1c50bb8f904a838f3/B of d6ca3b1ced864db1c50bb8f904a838f3 into b9d122b9ef0b4f6687361e34760d1d91(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:53:58,240 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:53:58,240 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., storeName=d6ca3b1ced864db1c50bb8f904a838f3/B, priority=13, startTime=1732776838086; duration=0sec 2024-11-28T06:53:58,241 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:53:58,241 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ca3b1ced864db1c50bb8f904a838f3:B 2024-11-28T06:53:58,241 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T06:53:58,246 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48326 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T06:53:58,246 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): d6ca3b1ced864db1c50bb8f904a838f3/C is initiating minor compaction (all files) 2024-11-28T06:53:58,246 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ca3b1ced864db1c50bb8f904a838f3/C in TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:58,247 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/5751bb49721c41288cd46ed0d62623dc, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/87bbb20d080c48498a0e5dea3eb7bdf6, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/c5be36c79319478db2d897a12d91904f, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/4b608daed9b24b46897dafb85f89087a] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp, totalSize=47.2 K 2024-11-28T06:53:58,247 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 5751bb49721c41288cd46ed0d62623dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732776834582 2024-11-28T06:53:58,248 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 87bbb20d080c48498a0e5dea3eb7bdf6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732776835893 2024-11-28T06:53:58,251 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting c5be36c79319478db2d897a12d91904f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1732776836102 2024-11-28T06:53:58,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/2b71029d0173461595bcdebc6fabfc41 is 50, key is test_row_0/B:col10/1732776838097/Put/seqid=0 2024-11-28T06:53:58,253 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b608daed9b24b46897dafb85f89087a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732776836848 2024-11-28T06:53:58,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T06:53:58,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:58,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:53:58,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:58,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:58,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:58,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:58,291 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ca3b1ced864db1c50bb8f904a838f3#C#compaction#33 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:53:58,292 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/216e3d654da84f169259099463922641 is 50, key is test_row_0/C:col10/1732776836859/Put/seqid=0 2024-11-28T06:53:58,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741869_1045 (size=12151) 2024-11-28T06:53:58,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:58,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776898306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:58,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:58,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:58,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:58,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:58,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776898306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:58,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:58,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:58,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:58,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:58,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776898306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:58,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:58,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:58,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:58,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:58,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776898319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:58,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:58,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:58,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:58,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741870_1046 (size=12459) 2024-11-28T06:53:58,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T06:53:58,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:58,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:53:58,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:58,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:58,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-28T06:53:58,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:58,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:58,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:58,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:58,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:58,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776898519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:58,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776898516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:58,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:58,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:58,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:58,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:58,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:58,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:58,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776898517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:58,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:58,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:58,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:58,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:58,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776898536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:58,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:58,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:58,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:58,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T06:53:58,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:58,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:53:58,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:58,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:58,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:58,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:58,634 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/a9e13f7e074441608fa9c7f6f7e00529 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/a9e13f7e074441608fa9c7f6f7e00529 2024-11-28T06:53:58,653 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ca3b1ced864db1c50bb8f904a838f3/A of d6ca3b1ced864db1c50bb8f904a838f3 into a9e13f7e074441608fa9c7f6f7e00529(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:53:58,653 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:53:58,653 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., storeName=d6ca3b1ced864db1c50bb8f904a838f3/A, priority=13, startTime=1732776838086; duration=0sec 2024-11-28T06:53:58,653 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:53:58,654 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ca3b1ced864db1c50bb8f904a838f3:A 2024-11-28T06:53:58,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/2b71029d0173461595bcdebc6fabfc41 2024-11-28T06:53:58,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/81aaead0fcd345cea9debd442ddaaa19 is 50, key is test_row_0/C:col10/1732776838097/Put/seqid=0 2024-11-28T06:53:58,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T06:53:58,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:58,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:53:58,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:58,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:58,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:58,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:58,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741871_1047 (size=12151) 2024-11-28T06:53:58,760 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/81aaead0fcd345cea9debd442ddaaa19 2024-11-28T06:53:58,776 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/216e3d654da84f169259099463922641 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/216e3d654da84f169259099463922641 2024-11-28T06:53:58,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/dd28d6c98e0d4e9286d6b71539737d42 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/dd28d6c98e0d4e9286d6b71539737d42 2024-11-28T06:53:58,789 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d6ca3b1ced864db1c50bb8f904a838f3/C of d6ca3b1ced864db1c50bb8f904a838f3 into 216e3d654da84f169259099463922641(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:53:58,790 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:53:58,790 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., storeName=d6ca3b1ced864db1c50bb8f904a838f3/C, priority=12, startTime=1732776838086; duration=0sec 2024-11-28T06:53:58,790 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:53:58,790 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ca3b1ced864db1c50bb8f904a838f3:C 2024-11-28T06:53:58,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/dd28d6c98e0d4e9286d6b71539737d42, entries=200, sequenceid=160, filesize=14.2 K 2024-11-28T06:53:58,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/2b71029d0173461595bcdebc6fabfc41 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/2b71029d0173461595bcdebc6fabfc41 2024-11-28T06:53:58,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/2b71029d0173461595bcdebc6fabfc41, entries=150, sequenceid=160, filesize=11.9 K 2024-11-28T06:53:58,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/81aaead0fcd345cea9debd442ddaaa19 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/81aaead0fcd345cea9debd442ddaaa19 2024-11-28T06:53:58,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/81aaead0fcd345cea9debd442ddaaa19, entries=150, sequenceid=160, filesize=11.9 K 2024-11-28T06:53:58,818 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for d6ca3b1ced864db1c50bb8f904a838f3 in 719ms, sequenceid=160, compaction requested=false 2024-11-28T06:53:58,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:53:58,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:53:58,835 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6ca3b1ced864db1c50bb8f904a838f3 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T06:53:58,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=A 2024-11-28T06:53:58,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:53:58,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=B 2024-11-28T06:53:58,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:53:58,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=C 2024-11-28T06:53:58,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:53:58,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/e3b35fd57dca43ce9b95ac510c8c73b0 is 50, key is test_row_0/A:col10/1732776838190/Put/seqid=0 2024-11-28T06:53:58,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:58,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776898872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:58,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:58,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:58,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:58,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T06:53:58,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:58,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:53:58,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:58,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:58,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:58,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:58,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:58,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776898882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:58,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776898876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:58,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:58,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:58,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:58,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776898891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:58,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:58,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:58,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:58,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:58,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:58,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:58,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:58,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:58,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741872_1048 (size=14541) 2024-11-28T06:53:58,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/e3b35fd57dca43ce9b95ac510c8c73b0 2024-11-28T06:53:58,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-28T06:53:58,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/4a33c7340fd84fb49a4b39f2f5c3cae4 is 50, key is test_row_0/B:col10/1732776838190/Put/seqid=0 2024-11-28T06:53:58,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741873_1049 (size=12151) 2024-11-28T06:53:58,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/4a33c7340fd84fb49a4b39f2f5c3cae4 2024-11-28T06:53:58,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/2a02f310a51145479068ab9daf0278a7 is 50, key is test_row_0/C:col10/1732776838190/Put/seqid=0 2024-11-28T06:53:58,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:58,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776898997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:58,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:58,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:59,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:59,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776899006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:59,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:59,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:59,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776899006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:59,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:59,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:59,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776899016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:59,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:59,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741874_1050 (size=12151) 2024-11-28T06:53:59,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T06:53:59,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:59,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:53:59,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:59,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:59,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:59,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:59,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:59,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776899206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:59,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:59,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T06:53:59,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:59,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:53:59,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:59,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:59,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:59,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:59,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:59,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776899216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:59,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:59,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:59,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776899216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:59,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:59,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:59,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776899226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:59,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:59,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T06:53:59,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:59,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:53:59,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:59,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:59,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:59,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:59,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/2a02f310a51145479068ab9daf0278a7 2024-11-28T06:53:59,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/e3b35fd57dca43ce9b95ac510c8c73b0 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/e3b35fd57dca43ce9b95ac510c8c73b0 2024-11-28T06:53:59,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/e3b35fd57dca43ce9b95ac510c8c73b0, entries=200, sequenceid=183, filesize=14.2 K 2024-11-28T06:53:59,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/4a33c7340fd84fb49a4b39f2f5c3cae4 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/4a33c7340fd84fb49a4b39f2f5c3cae4 2024-11-28T06:53:59,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/4a33c7340fd84fb49a4b39f2f5c3cae4, entries=150, sequenceid=183, filesize=11.9 K 2024-11-28T06:53:59,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/2a02f310a51145479068ab9daf0278a7 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/2a02f310a51145479068ab9daf0278a7 2024-11-28T06:53:59,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/2a02f310a51145479068ab9daf0278a7, entries=150, sequenceid=183, filesize=11.9 K 2024-11-28T06:53:59,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for d6ca3b1ced864db1c50bb8f904a838f3 in 659ms, sequenceid=183, compaction requested=true 2024-11-28T06:53:59,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:53:59,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ca3b1ced864db1c50bb8f904a838f3:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:53:59,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:53:59,495 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:53:59,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ca3b1ced864db1c50bb8f904a838f3:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:53:59,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:53:59,495 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:53:59,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ca3b1ced864db1c50bb8f904a838f3:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:53:59,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:53:59,497 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:53:59,497 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41575 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:53:59,497 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): d6ca3b1ced864db1c50bb8f904a838f3/B is initiating minor compaction (all files) 2024-11-28T06:53:59,497 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): d6ca3b1ced864db1c50bb8f904a838f3/A is initiating minor compaction (all files) 2024-11-28T06:53:59,497 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ca3b1ced864db1c50bb8f904a838f3/A in TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:59,497 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ca3b1ced864db1c50bb8f904a838f3/B in TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:59,497 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/b9d122b9ef0b4f6687361e34760d1d91, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/2b71029d0173461595bcdebc6fabfc41, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/4a33c7340fd84fb49a4b39f2f5c3cae4] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp, totalSize=35.9 K 2024-11-28T06:53:59,497 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/a9e13f7e074441608fa9c7f6f7e00529, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/dd28d6c98e0d4e9286d6b71539737d42, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/e3b35fd57dca43ce9b95ac510c8c73b0] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp, totalSize=40.6 K 2024-11-28T06:53:59,498 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting b9d122b9ef0b4f6687361e34760d1d91, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732776836848 2024-11-28T06:53:59,498 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9e13f7e074441608fa9c7f6f7e00529, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732776836848 2024-11-28T06:53:59,499 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd28d6c98e0d4e9286d6b71539737d42, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732776837524 2024-11-28T06:53:59,499 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b71029d0173461595bcdebc6fabfc41, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732776837524 2024-11-28T06:53:59,500 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3b35fd57dca43ce9b95ac510c8c73b0, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732776838185 2024-11-28T06:53:59,502 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a33c7340fd84fb49a4b39f2f5c3cae4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732776838185 2024-11-28T06:53:59,511 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ca3b1ced864db1c50bb8f904a838f3#A#compaction#38 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:53:59,512 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/af69f679760c4a4ebd8488592075933e is 50, key is test_row_0/A:col10/1732776838190/Put/seqid=0 2024-11-28T06:53:59,514 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ca3b1ced864db1c50bb8f904a838f3#B#compaction#39 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:53:59,515 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/f90f2c59fa4c4f549a84677e09cdc585 is 50, key is test_row_0/B:col10/1732776838190/Put/seqid=0 2024-11-28T06:53:59,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:53:59,523 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6ca3b1ced864db1c50bb8f904a838f3 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T06:53:59,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=A 2024-11-28T06:53:59,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:53:59,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=B 2024-11-28T06:53:59,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:53:59,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=C 2024-11-28T06:53:59,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:53:59,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T06:53:59,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:59,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:53:59,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:59,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:59,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:59,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:59,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/da856ae4fcc8463ea3ded63e13a2eb00 is 50, key is test_row_0/A:col10/1732776839522/Put/seqid=0 2024-11-28T06:53:59,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741875_1051 (size=12595) 2024-11-28T06:53:59,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741876_1052 (size=12595) 2024-11-28T06:53:59,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:59,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776899566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:59,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:59,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:59,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776899570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:59,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:59,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776899572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:59,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:59,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:59,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:59,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776899576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:59,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:59,584 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/f90f2c59fa4c4f549a84677e09cdc585 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/f90f2c59fa4c4f549a84677e09cdc585 2024-11-28T06:53:59,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741877_1053 (size=14541) 2024-11-28T06:53:59,595 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ca3b1ced864db1c50bb8f904a838f3/B of d6ca3b1ced864db1c50bb8f904a838f3 into f90f2c59fa4c4f549a84677e09cdc585(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:53:59,596 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:53:59,596 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., storeName=d6ca3b1ced864db1c50bb8f904a838f3/B, priority=13, startTime=1732776839495; duration=0sec 2024-11-28T06:53:59,596 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:53:59,596 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ca3b1ced864db1c50bb8f904a838f3:B 2024-11-28T06:53:59,596 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:53:59,598 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:53:59,598 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): d6ca3b1ced864db1c50bb8f904a838f3/C is initiating minor compaction (all files) 2024-11-28T06:53:59,598 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ca3b1ced864db1c50bb8f904a838f3/C in TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:59,598 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/216e3d654da84f169259099463922641, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/81aaead0fcd345cea9debd442ddaaa19, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/2a02f310a51145479068ab9daf0278a7] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp, totalSize=35.9 K 2024-11-28T06:53:59,600 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 216e3d654da84f169259099463922641, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732776836848 2024-11-28T06:53:59,601 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 81aaead0fcd345cea9debd442ddaaa19, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732776837524 2024-11-28T06:53:59,602 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a02f310a51145479068ab9daf0278a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732776838185 2024-11-28T06:53:59,624 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ca3b1ced864db1c50bb8f904a838f3#C#compaction#41 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:53:59,625 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/125c741f1a7849ac86e68ade93170a66 is 50, key is test_row_0/C:col10/1732776838190/Put/seqid=0 2024-11-28T06:53:59,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741878_1054 (size=12561) 2024-11-28T06:53:59,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T06:53:59,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:59,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:53:59,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:59,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:59,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:59,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:59,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:59,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776899686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:59,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:59,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776899686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:59,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:59,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776899687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:59,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:59,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:59,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:59,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:59,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776899690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:59,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:59,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T06:53:59,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:59,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:53:59,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:59,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:59,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:59,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:59,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:59,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776899897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:59,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:59,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:59,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776899897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:59,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:59,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776899897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:59,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:59,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:59,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:53:59,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776899905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:53:59,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:53:59,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:53:59,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-28T06:53:59,968 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/af69f679760c4a4ebd8488592075933e as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/af69f679760c4a4ebd8488592075933e 2024-11-28T06:53:59,989 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ca3b1ced864db1c50bb8f904a838f3/A of d6ca3b1ced864db1c50bb8f904a838f3 into af69f679760c4a4ebd8488592075933e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:53:59,989 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:53:59,989 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., storeName=d6ca3b1ced864db1c50bb8f904a838f3/A, priority=13, startTime=1732776839495; duration=0sec 2024-11-28T06:53:59,990 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:53:59,990 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ca3b1ced864db1c50bb8f904a838f3:A 2024-11-28T06:53:59,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T06:53:59,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:59,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:53:59,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:53:59,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:59,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:59,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:53:59,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/da856ae4fcc8463ea3ded63e13a2eb00 2024-11-28T06:54:00,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:00,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36870 deadline: 1732776900006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:00,013 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_0' in d6ca3b1ced864db1c50bb8f904a838f3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8382 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:54:00,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:00,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:00,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:00,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/63ef5eb3097f43b09cddb0cf36952969 is 50, key is test_row_0/B:col10/1732776839522/Put/seqid=0 2024-11-28T06:54:00,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741879_1055 (size=12151) 2024-11-28T06:54:00,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/63ef5eb3097f43b09cddb0cf36952969 2024-11-28T06:54:00,061 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/125c741f1a7849ac86e68ade93170a66 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/125c741f1a7849ac86e68ade93170a66 2024-11-28T06:54:00,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/6f3cff27e83b4604982f474f1e6b8851 is 50, key is test_row_0/C:col10/1732776839522/Put/seqid=0 2024-11-28T06:54:00,083 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ca3b1ced864db1c50bb8f904a838f3/C of d6ca3b1ced864db1c50bb8f904a838f3 into 125c741f1a7849ac86e68ade93170a66(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:00,083 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:54:00,084 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., storeName=d6ca3b1ced864db1c50bb8f904a838f3/C, priority=13, startTime=1732776839495; duration=0sec 2024-11-28T06:54:00,088 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:00,088 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ca3b1ced864db1c50bb8f904a838f3:C 2024-11-28T06:54:00,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741880_1056 (size=12151) 2024-11-28T06:54:00,112 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/6f3cff27e83b4604982f474f1e6b8851 2024-11-28T06:54:00,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/da856ae4fcc8463ea3ded63e13a2eb00 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/da856ae4fcc8463ea3ded63e13a2eb00 2024-11-28T06:54:00,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/da856ae4fcc8463ea3ded63e13a2eb00, entries=200, sequenceid=197, filesize=14.2 K 2024-11-28T06:54:00,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/63ef5eb3097f43b09cddb0cf36952969 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/63ef5eb3097f43b09cddb0cf36952969 2024-11-28T06:54:00,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T06:54:00,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:00,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:54:00,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:00,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:00,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:00,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:00,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/63ef5eb3097f43b09cddb0cf36952969, entries=150, sequenceid=197, filesize=11.9 K 2024-11-28T06:54:00,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/6f3cff27e83b4604982f474f1e6b8851 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/6f3cff27e83b4604982f474f1e6b8851 2024-11-28T06:54:00,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/6f3cff27e83b4604982f474f1e6b8851, entries=150, sequenceid=197, filesize=11.9 K 2024-11-28T06:54:00,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for d6ca3b1ced864db1c50bb8f904a838f3 in 642ms, sequenceid=197, compaction requested=false 2024-11-28T06:54:00,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:54:00,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:54:00,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6ca3b1ced864db1c50bb8f904a838f3 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T06:54:00,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=A 2024-11-28T06:54:00,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:00,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=B 2024-11-28T06:54:00,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:00,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=C 2024-11-28T06:54:00,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:00,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/5fb72f117b644c2681b18b93ec9fff02 is 50, key is test_row_0/A:col10/1732776840223/Put/seqid=0 2024-11-28T06:54:00,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:00,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776900254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:00,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:00,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:00,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:00,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:00,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776900259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:00,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:00,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:00,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:00,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:00,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:00,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776900259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:00,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776900261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:00,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:00,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:00,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:00,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:00,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:00,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:00,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741881_1057 (size=16931) 2024-11-28T06:54:00,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/5fb72f117b644c2681b18b93ec9fff02 2024-11-28T06:54:00,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T06:54:00,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:00,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:54:00,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:00,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:00,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:00,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:00,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/3360b9cc25b34267a770ef07a6f5df07 is 50, key is test_row_0/B:col10/1732776840223/Put/seqid=0 2024-11-28T06:54:00,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741882_1058 (size=12151) 2024-11-28T06:54:00,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/3360b9cc25b34267a770ef07a6f5df07 2024-11-28T06:54:00,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:00,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776900375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:00,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:00,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776900376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:00,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:00,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:00,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:00,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:00,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:00,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:00,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:00,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776900376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:00,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:00,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:00,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:00,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:00,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776900385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:00,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:00,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:00,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:00,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/cf3f7722dba946ffa5516cbff77db054 is 50, key is test_row_0/C:col10/1732776840223/Put/seqid=0 2024-11-28T06:54:00,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741883_1059 (size=12151) 2024-11-28T06:54:00,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/cf3f7722dba946ffa5516cbff77db054 2024-11-28T06:54:00,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/5fb72f117b644c2681b18b93ec9fff02 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/5fb72f117b644c2681b18b93ec9fff02 2024-11-28T06:54:00,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T06:54:00,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:00,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:54:00,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:00,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:00,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:00,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:00,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/5fb72f117b644c2681b18b93ec9fff02, entries=250, sequenceid=220, filesize=16.5 K 2024-11-28T06:54:00,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/3360b9cc25b34267a770ef07a6f5df07 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/3360b9cc25b34267a770ef07a6f5df07 2024-11-28T06:54:00,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/3360b9cc25b34267a770ef07a6f5df07, entries=150, sequenceid=220, filesize=11.9 K 2024-11-28T06:54:00,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/cf3f7722dba946ffa5516cbff77db054 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/cf3f7722dba946ffa5516cbff77db054 2024-11-28T06:54:00,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/cf3f7722dba946ffa5516cbff77db054, entries=150, sequenceid=220, filesize=11.9 K 2024-11-28T06:54:00,519 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for d6ca3b1ced864db1c50bb8f904a838f3 in 292ms, sequenceid=220, compaction requested=true 2024-11-28T06:54:00,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:54:00,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ca3b1ced864db1c50bb8f904a838f3:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:54:00,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:00,519 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:54:00,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ca3b1ced864db1c50bb8f904a838f3:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:54:00,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:00,519 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:54:00,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ca3b1ced864db1c50bb8f904a838f3:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:54:00,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:54:00,521 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44067 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:54:00,521 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:54:00,521 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): d6ca3b1ced864db1c50bb8f904a838f3/B is initiating minor compaction (all files) 2024-11-28T06:54:00,521 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): d6ca3b1ced864db1c50bb8f904a838f3/A is initiating minor compaction (all files) 2024-11-28T06:54:00,521 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ca3b1ced864db1c50bb8f904a838f3/A in TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:00,522 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/af69f679760c4a4ebd8488592075933e, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/da856ae4fcc8463ea3ded63e13a2eb00, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/5fb72f117b644c2681b18b93ec9fff02] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp, totalSize=43.0 K 2024-11-28T06:54:00,522 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ca3b1ced864db1c50bb8f904a838f3/B in TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:00,522 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/f90f2c59fa4c4f549a84677e09cdc585, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/63ef5eb3097f43b09cddb0cf36952969, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/3360b9cc25b34267a770ef07a6f5df07] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp, totalSize=36.0 K 2024-11-28T06:54:00,522 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting af69f679760c4a4ebd8488592075933e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732776838185 2024-11-28T06:54:00,523 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting f90f2c59fa4c4f549a84677e09cdc585, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732776838185 2024-11-28T06:54:00,523 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting da856ae4fcc8463ea3ded63e13a2eb00, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732776838860 2024-11-28T06:54:00,524 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fb72f117b644c2681b18b93ec9fff02, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732776839539 2024-11-28T06:54:00,524 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 63ef5eb3097f43b09cddb0cf36952969, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732776838860 2024-11-28T06:54:00,537 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 3360b9cc25b34267a770ef07a6f5df07, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732776839572 2024-11-28T06:54:00,557 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ca3b1ced864db1c50bb8f904a838f3#A#compaction#47 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:54:00,558 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/2d8f7389c7d8493e87f24bc6ad773dc7 is 50, key is test_row_0/A:col10/1732776840223/Put/seqid=0 2024-11-28T06:54:00,571 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ca3b1ced864db1c50bb8f904a838f3#B#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:54:00,572 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/30bd06de75f14d5ca8f6947d833f5042 is 50, key is test_row_0/B:col10/1732776840223/Put/seqid=0 2024-11-28T06:54:00,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:54:00,599 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6ca3b1ced864db1c50bb8f904a838f3 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T06:54:00,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=A 2024-11-28T06:54:00,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:00,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=B 2024-11-28T06:54:00,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:00,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=C 2024-11-28T06:54:00,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:00,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741884_1060 (size=12697) 2024-11-28T06:54:00,618 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/2d8f7389c7d8493e87f24bc6ad773dc7 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/2d8f7389c7d8493e87f24bc6ad773dc7 2024-11-28T06:54:00,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T06:54:00,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:00,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:54:00,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:00,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:00,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:00,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:00,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741885_1061 (size=12697) 2024-11-28T06:54:00,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/5f453a0e67ca4fc08c34db296ba72a97 is 50, key is test_row_0/A:col10/1732776840253/Put/seqid=0 2024-11-28T06:54:00,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741886_1062 (size=12151) 2024-11-28T06:54:00,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/5f453a0e67ca4fc08c34db296ba72a97 2024-11-28T06:54:00,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/d04e4c06476041d5a4ed53a4e5467be3 is 50, key is test_row_0/B:col10/1732776840253/Put/seqid=0 2024-11-28T06:54:00,679 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ca3b1ced864db1c50bb8f904a838f3/A of d6ca3b1ced864db1c50bb8f904a838f3 into 2d8f7389c7d8493e87f24bc6ad773dc7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:00,679 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:54:00,679 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., storeName=d6ca3b1ced864db1c50bb8f904a838f3/A, priority=13, startTime=1732776840519; duration=0sec 2024-11-28T06:54:00,679 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:54:00,679 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ca3b1ced864db1c50bb8f904a838f3:A 2024-11-28T06:54:00,679 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:54:00,681 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:54:00,682 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): d6ca3b1ced864db1c50bb8f904a838f3/C is initiating minor compaction (all files) 2024-11-28T06:54:00,682 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ca3b1ced864db1c50bb8f904a838f3/C in TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:00,682 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/125c741f1a7849ac86e68ade93170a66, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/6f3cff27e83b4604982f474f1e6b8851, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/cf3f7722dba946ffa5516cbff77db054] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp, totalSize=36.0 K 2024-11-28T06:54:00,683 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 125c741f1a7849ac86e68ade93170a66, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732776838185 2024-11-28T06:54:00,683 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f3cff27e83b4604982f474f1e6b8851, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732776838860 2024-11-28T06:54:00,683 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf3f7722dba946ffa5516cbff77db054, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732776839572 2024-11-28T06:54:00,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741887_1063 (size=12151) 2024-11-28T06:54:00,701 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ca3b1ced864db1c50bb8f904a838f3#C#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:54:00,701 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/fce399ca4f334463aa825024a75e3c58 is 50, key is test_row_0/C:col10/1732776840223/Put/seqid=0 2024-11-28T06:54:00,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/d04e4c06476041d5a4ed53a4e5467be3 2024-11-28T06:54:00,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741888_1064 (size=12663) 2024-11-28T06:54:00,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:00,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776900708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:00,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:00,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:00,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:00,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776900708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:00,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:00,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:00,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:00,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:00,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:00,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776900710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:00,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:00,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:00,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:00,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:00,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776900717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:00,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:00,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:00,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:00,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/c3303330af7b439b95dde2ac665a8393 is 50, key is test_row_0/C:col10/1732776840253/Put/seqid=0 2024-11-28T06:54:00,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741889_1065 (size=12151) 2024-11-28T06:54:00,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T06:54:00,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:00,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:54:00,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:00,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:00,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:00,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:00,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:00,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776900826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:00,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:00,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:00,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:00,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:00,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776900826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:00,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:00,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:00,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:00,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:00,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776900826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:00,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:00,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:00,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:00,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:00,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776900826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:00,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:00,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:00,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:00,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T06:54:00,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:00,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:54:00,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:00,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:00,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:00,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:01,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:01,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776901036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:01,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:01,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:01,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776901036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:01,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:01,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776901036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:01,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:01,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:01,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:01,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776901036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:01,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:01,055 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/30bd06de75f14d5ca8f6947d833f5042 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/30bd06de75f14d5ca8f6947d833f5042 2024-11-28T06:54:01,069 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ca3b1ced864db1c50bb8f904a838f3/B of d6ca3b1ced864db1c50bb8f904a838f3 into 30bd06de75f14d5ca8f6947d833f5042(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:01,069 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:54:01,069 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., storeName=d6ca3b1ced864db1c50bb8f904a838f3/B, priority=13, startTime=1732776840519; duration=0sec 2024-11-28T06:54:01,069 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:01,069 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ca3b1ced864db1c50bb8f904a838f3:B 2024-11-28T06:54:01,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T06:54:01,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:01,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:54:01,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:01,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:01,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:01,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:01,131 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/fce399ca4f334463aa825024a75e3c58 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/fce399ca4f334463aa825024a75e3c58 2024-11-28T06:54:01,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/c3303330af7b439b95dde2ac665a8393 2024-11-28T06:54:01,142 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ca3b1ced864db1c50bb8f904a838f3/C of d6ca3b1ced864db1c50bb8f904a838f3 into fce399ca4f334463aa825024a75e3c58(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:01,142 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:54:01,142 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., storeName=d6ca3b1ced864db1c50bb8f904a838f3/C, priority=13, startTime=1732776840519; duration=0sec 2024-11-28T06:54:01,142 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:01,142 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ca3b1ced864db1c50bb8f904a838f3:C 2024-11-28T06:54:01,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/5f453a0e67ca4fc08c34db296ba72a97 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/5f453a0e67ca4fc08c34db296ba72a97 2024-11-28T06:54:01,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/5f453a0e67ca4fc08c34db296ba72a97, entries=150, sequenceid=233, filesize=11.9 K 2024-11-28T06:54:01,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/d04e4c06476041d5a4ed53a4e5467be3 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/d04e4c06476041d5a4ed53a4e5467be3 2024-11-28T06:54:01,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/d04e4c06476041d5a4ed53a4e5467be3, entries=150, sequenceid=233, filesize=11.9 K 2024-11-28T06:54:01,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/c3303330af7b439b95dde2ac665a8393 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/c3303330af7b439b95dde2ac665a8393 2024-11-28T06:54:01,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/c3303330af7b439b95dde2ac665a8393, entries=150, sequenceid=233, filesize=11.9 K 2024-11-28T06:54:01,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for d6ca3b1ced864db1c50bb8f904a838f3 in 611ms, sequenceid=233, compaction requested=false 2024-11-28T06:54:01,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:54:01,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T06:54:01,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:01,250 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing d6ca3b1ced864db1c50bb8f904a838f3 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T06:54:01,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=A 2024-11-28T06:54:01,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:01,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=B 2024-11-28T06:54:01,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:01,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=C 2024-11-28T06:54:01,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:01,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/952ec844c1214ce6be5d7f0af5060bb0 is 50, key is test_row_0/A:col10/1732776840714/Put/seqid=0 2024-11-28T06:54:01,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741890_1066 (size=12151) 2024-11-28T06:54:01,324 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/952ec844c1214ce6be5d7f0af5060bb0 2024-11-28T06:54:01,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/df5a104f58ad4f469e9522e7534c138f is 50, key is test_row_0/B:col10/1732776840714/Put/seqid=0 2024-11-28T06:54:01,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:54:01,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:54:01,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741891_1067 (size=12151) 2024-11-28T06:54:01,373 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/df5a104f58ad4f469e9522e7534c138f 2024-11-28T06:54:01,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:01,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776901366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:01,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:01,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:01,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776901374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:01,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:01,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776901375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:01,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:01,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:01,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776901379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:01,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:01,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:01,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/4fbc6f7e0f2d46209cf48ffaa9d1c52f is 50, key is test_row_0/C:col10/1732776840714/Put/seqid=0 2024-11-28T06:54:01,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741892_1068 (size=12151) 2024-11-28T06:54:01,398 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/4fbc6f7e0f2d46209cf48ffaa9d1c52f 2024-11-28T06:54:01,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/952ec844c1214ce6be5d7f0af5060bb0 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/952ec844c1214ce6be5d7f0af5060bb0 2024-11-28T06:54:01,421 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/952ec844c1214ce6be5d7f0af5060bb0, entries=150, sequenceid=255, filesize=11.9 K 2024-11-28T06:54:01,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/df5a104f58ad4f469e9522e7534c138f as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/df5a104f58ad4f469e9522e7534c138f 2024-11-28T06:54:01,433 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/df5a104f58ad4f469e9522e7534c138f, entries=150, sequenceid=255, filesize=11.9 K 2024-11-28T06:54:01,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/4fbc6f7e0f2d46209cf48ffaa9d1c52f as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/4fbc6f7e0f2d46209cf48ffaa9d1c52f 2024-11-28T06:54:01,443 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/4fbc6f7e0f2d46209cf48ffaa9d1c52f, entries=150, sequenceid=255, filesize=11.9 K 2024-11-28T06:54:01,444 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for d6ca3b1ced864db1c50bb8f904a838f3 in 194ms, sequenceid=255, compaction requested=true 2024-11-28T06:54:01,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:54:01,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:01,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-28T06:54:01,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-28T06:54:01,456 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-28T06:54:01,456 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.6530 sec 2024-11-28T06:54:01,460 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 3.6700 sec 2024-11-28T06:54:01,489 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6ca3b1ced864db1c50bb8f904a838f3 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T06:54:01,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=A 2024-11-28T06:54:01,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:01,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=B 2024-11-28T06:54:01,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:01,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=C 2024-11-28T06:54:01,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:01,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:54:01,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/2dcc9874f8a142f4a8a91ecb5f77a169 is 50, key is test_row_0/A:col10/1732776841487/Put/seqid=0 2024-11-28T06:54:01,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:01,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776901516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:01,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:01,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:01,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776901518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:01,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:01,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:01,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776901522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:01,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:01,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:01,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776901527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:01,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:01,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741893_1069 (size=14741) 2024-11-28T06:54:01,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:01,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776901637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:01,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:01,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:01,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776901637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:01,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776901637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:01,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:01,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:01,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776901637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:01,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:01,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:01,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:01,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:01,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776901845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:01,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:01,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776901845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:01,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:01,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:01,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:01,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776901846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:01,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:01,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:01,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776901856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:01,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:01,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/2dcc9874f8a142f4a8a91ecb5f77a169 2024-11-28T06:54:01,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/af78906486b84ab4aad8346befaedbe1 is 50, key is test_row_0/B:col10/1732776841487/Put/seqid=0 2024-11-28T06:54:01,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-28T06:54:01,956 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:54:01,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741894_1070 (size=12301) 2024-11-28T06:54:01,957 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:54:01,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-28T06:54:01,959 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:54:01,960 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:54:01,960 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:54:01,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T06:54:02,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T06:54:02,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-28T06:54:02,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:02,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:54:02,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:02,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:02,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:02,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:02,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:02,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776902156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:02,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:02,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776902157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:02,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:02,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:02,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:02,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:02,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:02,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:02,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:02,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776902166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:02,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:02,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:02,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:02,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776902166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:02,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:02,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:02,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:02,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:02,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-28T06:54:02,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:02,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:54:02,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:02,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:02,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:02,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:02,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T06:54:02,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/af78906486b84ab4aad8346befaedbe1 2024-11-28T06:54:02,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/301e23cfb9014642a998017b57fd1101 is 50, key is test_row_0/C:col10/1732776841487/Put/seqid=0 2024-11-28T06:54:02,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741895_1071 (size=12301) 2024-11-28T06:54:02,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-28T06:54:02,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:02,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:54:02,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:02,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:02,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:02,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:02,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-28T06:54:02,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:02,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:54:02,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:02,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:02,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:02,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:02,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T06:54:02,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:02,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:02,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776902666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:02,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776902666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:02,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:02,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:02,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:02,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:02,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:02,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:02,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:02,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776902675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:02,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:02,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:02,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:02,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:02,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776902686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:02,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:02,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:02,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:02,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-28T06:54:02,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:02,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:54:02,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:02,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:02,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:02,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:02,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/301e23cfb9014642a998017b57fd1101 2024-11-28T06:54:02,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/2dcc9874f8a142f4a8a91ecb5f77a169 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/2dcc9874f8a142f4a8a91ecb5f77a169 2024-11-28T06:54:02,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/2dcc9874f8a142f4a8a91ecb5f77a169, entries=200, sequenceid=269, filesize=14.4 K 2024-11-28T06:54:02,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/af78906486b84ab4aad8346befaedbe1 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/af78906486b84ab4aad8346befaedbe1 2024-11-28T06:54:02,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/af78906486b84ab4aad8346befaedbe1, entries=150, sequenceid=269, filesize=12.0 K 2024-11-28T06:54:02,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/301e23cfb9014642a998017b57fd1101 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/301e23cfb9014642a998017b57fd1101 2024-11-28T06:54:02,819 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/301e23cfb9014642a998017b57fd1101, entries=150, sequenceid=269, filesize=12.0 K 2024-11-28T06:54:02,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for d6ca3b1ced864db1c50bb8f904a838f3 in 1333ms, sequenceid=269, compaction requested=true 2024-11-28T06:54:02,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:54:02,822 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T06:54:02,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ca3b1ced864db1c50bb8f904a838f3:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:54:02,823 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51740 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T06:54:02,824 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): d6ca3b1ced864db1c50bb8f904a838f3/A is initiating minor compaction (all files) 2024-11-28T06:54:02,824 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ca3b1ced864db1c50bb8f904a838f3/A in TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:02,824 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/2d8f7389c7d8493e87f24bc6ad773dc7, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/5f453a0e67ca4fc08c34db296ba72a97, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/952ec844c1214ce6be5d7f0af5060bb0, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/2dcc9874f8a142f4a8a91ecb5f77a169] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp, totalSize=50.5 K 2024-11-28T06:54:02,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:02,825 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d8f7389c7d8493e87f24bc6ad773dc7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732776839572 2024-11-28T06:54:02,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ca3b1ced864db1c50bb8f904a838f3:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:54:02,825 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f453a0e67ca4fc08c34db296ba72a97, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732776840253 2024-11-28T06:54:02,825 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T06:54:02,826 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 952ec844c1214ce6be5d7f0af5060bb0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732776840705 2024-11-28T06:54:02,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:02,827 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2dcc9874f8a142f4a8a91ecb5f77a169, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732776841360 2024-11-28T06:54:02,827 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49300 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T06:54:02,827 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): d6ca3b1ced864db1c50bb8f904a838f3/B is initiating minor compaction (all files) 2024-11-28T06:54:02,827 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ca3b1ced864db1c50bb8f904a838f3/B in TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:02,827 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/30bd06de75f14d5ca8f6947d833f5042, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/d04e4c06476041d5a4ed53a4e5467be3, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/df5a104f58ad4f469e9522e7534c138f, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/af78906486b84ab4aad8346befaedbe1] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp, totalSize=48.1 K 2024-11-28T06:54:02,828 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 30bd06de75f14d5ca8f6947d833f5042, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732776839572 2024-11-28T06:54:02,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ca3b1ced864db1c50bb8f904a838f3:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:54:02,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:54:02,831 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting d04e4c06476041d5a4ed53a4e5467be3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732776840253 2024-11-28T06:54:02,833 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting df5a104f58ad4f469e9522e7534c138f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732776840705 2024-11-28T06:54:02,835 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting af78906486b84ab4aad8346befaedbe1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732776841360 2024-11-28T06:54:02,858 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ca3b1ced864db1c50bb8f904a838f3#A#compaction#59 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:54:02,859 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/4da13ca3cd584ecfacb1ea7666e3beb7 is 50, key is test_row_0/A:col10/1732776841487/Put/seqid=0 2024-11-28T06:54:02,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741896_1072 (size=12983) 2024-11-28T06:54:02,885 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ca3b1ced864db1c50bb8f904a838f3#B#compaction#60 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:54:02,885 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/10b6957509f4415395e42d3f83b256e1 is 50, key is test_row_0/B:col10/1732776841487/Put/seqid=0 2024-11-28T06:54:02,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-28T06:54:02,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:02,886 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing d6ca3b1ced864db1c50bb8f904a838f3 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T06:54:02,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=A 2024-11-28T06:54:02,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:02,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=B 2024-11-28T06:54:02,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:02,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=C 2024-11-28T06:54:02,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:02,887 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/4da13ca3cd584ecfacb1ea7666e3beb7 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/4da13ca3cd584ecfacb1ea7666e3beb7 2024-11-28T06:54:02,903 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d6ca3b1ced864db1c50bb8f904a838f3/A of d6ca3b1ced864db1c50bb8f904a838f3 into 4da13ca3cd584ecfacb1ea7666e3beb7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:02,903 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:54:02,903 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., storeName=d6ca3b1ced864db1c50bb8f904a838f3/A, priority=12, startTime=1732776842821; duration=0sec 2024-11-28T06:54:02,903 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:54:02,903 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ca3b1ced864db1c50bb8f904a838f3:A 2024-11-28T06:54:02,903 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T06:54:02,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741897_1073 (size=12983) 2024-11-28T06:54:02,921 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T06:54:02,922 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): d6ca3b1ced864db1c50bb8f904a838f3/C is initiating minor compaction (all files) 2024-11-28T06:54:02,922 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ca3b1ced864db1c50bb8f904a838f3/C in TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:02,922 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/fce399ca4f334463aa825024a75e3c58, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/c3303330af7b439b95dde2ac665a8393, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/4fbc6f7e0f2d46209cf48ffaa9d1c52f, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/301e23cfb9014642a998017b57fd1101] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp, totalSize=48.1 K 2024-11-28T06:54:02,922 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting fce399ca4f334463aa825024a75e3c58, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732776839572 2024-11-28T06:54:02,923 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3303330af7b439b95dde2ac665a8393, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732776840253 2024-11-28T06:54:02,925 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fbc6f7e0f2d46209cf48ffaa9d1c52f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732776840705 2024-11-28T06:54:02,925 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 301e23cfb9014642a998017b57fd1101, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732776841360 2024-11-28T06:54:02,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/97cdfe8ad91e40e8b0015b19d6029c00 is 50, key is test_row_0/A:col10/1732776841516/Put/seqid=0 2024-11-28T06:54:02,940 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/10b6957509f4415395e42d3f83b256e1 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/10b6957509f4415395e42d3f83b256e1 2024-11-28T06:54:02,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741898_1074 (size=12301) 2024-11-28T06:54:02,952 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/97cdfe8ad91e40e8b0015b19d6029c00 2024-11-28T06:54:02,955 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ca3b1ced864db1c50bb8f904a838f3#C#compaction#62 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:54:02,956 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/42df70b67fa14c0ab5e72a6e2fba70a0 is 50, key is test_row_0/C:col10/1732776841487/Put/seqid=0 2024-11-28T06:54:02,964 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d6ca3b1ced864db1c50bb8f904a838f3/B of d6ca3b1ced864db1c50bb8f904a838f3 into 10b6957509f4415395e42d3f83b256e1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:02,964 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:54:02,965 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., storeName=d6ca3b1ced864db1c50bb8f904a838f3/B, priority=12, startTime=1732776842825; duration=0sec 2024-11-28T06:54:02,970 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:02,970 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ca3b1ced864db1c50bb8f904a838f3:B 2024-11-28T06:54:02,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741899_1075 (size=12949) 2024-11-28T06:54:02,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/d2c057985dd545ffa797712af3fe546c is 50, key is test_row_0/B:col10/1732776841516/Put/seqid=0 2024-11-28T06:54:02,996 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/42df70b67fa14c0ab5e72a6e2fba70a0 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/42df70b67fa14c0ab5e72a6e2fba70a0 2024-11-28T06:54:03,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741900_1076 (size=12301) 2024-11-28T06:54:03,009 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d6ca3b1ced864db1c50bb8f904a838f3/C of d6ca3b1ced864db1c50bb8f904a838f3 into 42df70b67fa14c0ab5e72a6e2fba70a0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:03,009 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:54:03,009 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., storeName=d6ca3b1ced864db1c50bb8f904a838f3/C, priority=12, startTime=1732776842828; duration=0sec 2024-11-28T06:54:03,009 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:03,009 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ca3b1ced864db1c50bb8f904a838f3:C 2024-11-28T06:54:03,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T06:54:03,402 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/d2c057985dd545ffa797712af3fe546c 2024-11-28T06:54:03,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/5dff3b621a864d57af9a1383acbd1b90 is 50, key is test_row_0/C:col10/1732776841516/Put/seqid=0 2024-11-28T06:54:03,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741901_1077 (size=12301) 2024-11-28T06:54:03,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:54:03,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:54:03,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:03,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776903744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:03,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:03,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776903746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:03,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:03,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:03,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:03,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:03,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:03,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:03,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:03,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776903748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:03,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:03,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:03,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:03,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:03,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776903749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:03,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:03,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:03,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:03,821 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/5dff3b621a864d57af9a1383acbd1b90 2024-11-28T06:54:03,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/97cdfe8ad91e40e8b0015b19d6029c00 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/97cdfe8ad91e40e8b0015b19d6029c00 2024-11-28T06:54:03,835 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/97cdfe8ad91e40e8b0015b19d6029c00, entries=150, sequenceid=287, filesize=12.0 K 2024-11-28T06:54:03,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/d2c057985dd545ffa797712af3fe546c as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/d2c057985dd545ffa797712af3fe546c 2024-11-28T06:54:03,841 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/d2c057985dd545ffa797712af3fe546c, entries=150, sequenceid=287, filesize=12.0 K 2024-11-28T06:54:03,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/5dff3b621a864d57af9a1383acbd1b90 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/5dff3b621a864d57af9a1383acbd1b90 2024-11-28T06:54:03,848 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/5dff3b621a864d57af9a1383acbd1b90, entries=150, sequenceid=287, filesize=12.0 K 2024-11-28T06:54:03,849 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for d6ca3b1ced864db1c50bb8f904a838f3 in 963ms, sequenceid=287, compaction requested=false 2024-11-28T06:54:03,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:54:03,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:03,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-28T06:54:03,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-28T06:54:03,853 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-28T06:54:03,854 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8910 sec 2024-11-28T06:54:03,857 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.8970 sec 2024-11-28T06:54:03,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:54:03,858 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6ca3b1ced864db1c50bb8f904a838f3 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-28T06:54:03,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=A 2024-11-28T06:54:03,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:03,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=B 2024-11-28T06:54:03,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:03,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=C 2024-11-28T06:54:03,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:03,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/a9ccb262218347de93b04e70f07f6a2f is 50, key is test_row_0/A:col10/1732776843857/Put/seqid=0 2024-11-28T06:54:03,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741902_1078 (size=17181) 2024-11-28T06:54:03,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:03,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776903885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:03,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:03,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:03,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776903887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:03,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:03,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:03,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:03,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:03,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:03,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:03,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776903891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:03,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:03,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:03,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:03,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:03,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776903893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:03,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:03,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:03,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:04,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:04,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:04,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776903995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:04,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776903995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:04,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:04,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:04,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:04,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:04,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:04,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:04,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:04,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776904006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:04,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:04,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:04,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:04,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:04,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776904006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:04,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:04,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:04,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:04,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T06:54:04,106 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:54:04,107 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:54:04,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-28T06:54:04,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-28T06:54:04,109 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:54:04,110 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:54:04,110 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:54:04,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:04,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776904205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:04,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:04,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776904206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:04,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:04,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:04,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:04,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:04,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:04,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:04,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-28T06:54:04,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:04,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776904216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:04,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:04,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:04,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:04,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:04,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776904216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:04,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:04,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:04,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:04,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-28T06:54:04,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:04,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:54:04,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:04,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:04,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:04,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:04,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/a9ccb262218347de93b04e70f07f6a2f 2024-11-28T06:54:04,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/5c738685dd644f2187993f39c4990e6f is 50, key is test_row_0/B:col10/1732776843857/Put/seqid=0 2024-11-28T06:54:04,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741903_1079 (size=12301) 2024-11-28T06:54:04,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-28T06:54:04,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:04,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:54:04,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:04,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:04,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:04,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:04,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-28T06:54:04,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:04,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776904522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:04,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:04,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776904522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:04,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:04,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:04,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:04,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:04,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:04,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:04,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:04,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776904525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:04,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:04,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776904526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:04,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:04,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:04,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:04,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:04,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:04,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:04,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-28T06:54:04,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:04,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:54:04,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:04,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:04,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:04,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:04,700 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/5c738685dd644f2187993f39c4990e6f 2024-11-28T06:54:04,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/4540486ce28c4cca869cee5e2083afd4 is 50, key is test_row_0/C:col10/1732776843857/Put/seqid=0 2024-11-28T06:54:04,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741904_1080 (size=12301) 2024-11-28T06:54:04,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-28T06:54:04,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:04,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:54:04,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:04,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:04,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:04,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:04,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-28T06:54:04,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-28T06:54:04,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:04,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:54:04,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:04,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:04,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:04,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:05,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-28T06:54:05,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:05,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:54:05,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:05,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:05,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:05,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:05,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:05,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776905036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:05,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:05,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:05,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:05,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:05,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776905036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:05,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:05,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776905036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:05,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:05,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776905036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:05,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:05,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:05,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:05,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:05,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:05,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:05,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:05,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:05,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:05,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/4540486ce28c4cca869cee5e2083afd4 2024-11-28T06:54:05,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/a9ccb262218347de93b04e70f07f6a2f as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/a9ccb262218347de93b04e70f07f6a2f 2024-11-28T06:54:05,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/a9ccb262218347de93b04e70f07f6a2f, entries=250, sequenceid=307, filesize=16.8 K 2024-11-28T06:54:05,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/5c738685dd644f2187993f39c4990e6f as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/5c738685dd644f2187993f39c4990e6f 2024-11-28T06:54:05,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/5c738685dd644f2187993f39c4990e6f, entries=150, sequenceid=307, filesize=12.0 K 2024-11-28T06:54:05,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/4540486ce28c4cca869cee5e2083afd4 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/4540486ce28c4cca869cee5e2083afd4 2024-11-28T06:54:05,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/4540486ce28c4cca869cee5e2083afd4, entries=150, sequenceid=307, filesize=12.0 K 2024-11-28T06:54:05,143 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for d6ca3b1ced864db1c50bb8f904a838f3 in 1285ms, sequenceid=307, compaction requested=true 2024-11-28T06:54:05,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:54:05,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ca3b1ced864db1c50bb8f904a838f3:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:54:05,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:05,143 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:54:05,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ca3b1ced864db1c50bb8f904a838f3:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:54:05,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:05,143 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:54:05,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ca3b1ced864db1c50bb8f904a838f3:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:54:05,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:54:05,144 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42465 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:54:05,144 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:54:05,144 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): d6ca3b1ced864db1c50bb8f904a838f3/A is initiating minor compaction (all files) 2024-11-28T06:54:05,144 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): d6ca3b1ced864db1c50bb8f904a838f3/B is initiating minor compaction (all files) 2024-11-28T06:54:05,145 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ca3b1ced864db1c50bb8f904a838f3/A in TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:05,145 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ca3b1ced864db1c50bb8f904a838f3/B in TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:05,145 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/10b6957509f4415395e42d3f83b256e1, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/d2c057985dd545ffa797712af3fe546c, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/5c738685dd644f2187993f39c4990e6f] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp, totalSize=36.7 K 2024-11-28T06:54:05,145 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/4da13ca3cd584ecfacb1ea7666e3beb7, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/97cdfe8ad91e40e8b0015b19d6029c00, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/a9ccb262218347de93b04e70f07f6a2f] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp, totalSize=41.5 K 2024-11-28T06:54:05,145 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4da13ca3cd584ecfacb1ea7666e3beb7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732776841360 2024-11-28T06:54:05,145 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 10b6957509f4415395e42d3f83b256e1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732776841360 2024-11-28T06:54:05,145 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting d2c057985dd545ffa797712af3fe546c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732776841505 2024-11-28T06:54:05,145 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97cdfe8ad91e40e8b0015b19d6029c00, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732776841505 2024-11-28T06:54:05,147 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c738685dd644f2187993f39c4990e6f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732776843745 2024-11-28T06:54:05,147 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9ccb262218347de93b04e70f07f6a2f, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732776843739 2024-11-28T06:54:05,156 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ca3b1ced864db1c50bb8f904a838f3#B#compaction#68 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:54:05,157 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/bcdabece3c6440afb7ffaea818e5707e is 50, key is test_row_0/B:col10/1732776843857/Put/seqid=0 2024-11-28T06:54:05,160 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ca3b1ced864db1c50bb8f904a838f3#A#compaction#69 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:54:05,160 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/3445a1f8c1d948838ac0f2e743201542 is 50, key is test_row_0/A:col10/1732776843857/Put/seqid=0 2024-11-28T06:54:05,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741906_1082 (size=13085) 2024-11-28T06:54:05,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741905_1081 (size=13085) 2024-11-28T06:54:05,173 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/3445a1f8c1d948838ac0f2e743201542 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/3445a1f8c1d948838ac0f2e743201542 2024-11-28T06:54:05,180 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ca3b1ced864db1c50bb8f904a838f3/A of d6ca3b1ced864db1c50bb8f904a838f3 into 3445a1f8c1d948838ac0f2e743201542(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:05,181 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:54:05,181 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., storeName=d6ca3b1ced864db1c50bb8f904a838f3/A, priority=13, startTime=1732776845143; duration=0sec 2024-11-28T06:54:05,181 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:54:05,181 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ca3b1ced864db1c50bb8f904a838f3:A 2024-11-28T06:54:05,181 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:54:05,184 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:54:05,184 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): d6ca3b1ced864db1c50bb8f904a838f3/C is initiating minor compaction (all files) 2024-11-28T06:54:05,185 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ca3b1ced864db1c50bb8f904a838f3/C in TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:05,185 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/42df70b67fa14c0ab5e72a6e2fba70a0, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/5dff3b621a864d57af9a1383acbd1b90, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/4540486ce28c4cca869cee5e2083afd4] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp, totalSize=36.7 K 2024-11-28T06:54:05,185 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42df70b67fa14c0ab5e72a6e2fba70a0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732776841360 2024-11-28T06:54:05,186 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5dff3b621a864d57af9a1383acbd1b90, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732776841505 2024-11-28T06:54:05,187 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4540486ce28c4cca869cee5e2083afd4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732776843745 2024-11-28T06:54:05,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-28T06:54:05,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:05,189 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing d6ca3b1ced864db1c50bb8f904a838f3 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-28T06:54:05,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=A 2024-11-28T06:54:05,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:05,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=B 2024-11-28T06:54:05,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:05,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=C 2024-11-28T06:54:05,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:05,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/a488ef25bf78450098e75eace8093d36 is 50, key is test_row_0/A:col10/1732776843875/Put/seqid=0 2024-11-28T06:54:05,205 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ca3b1ced864db1c50bb8f904a838f3#C#compaction#71 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:54:05,205 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/eeed1ec3f89243b3a1b5d6b46a54fee4 is 50, key is test_row_0/C:col10/1732776843857/Put/seqid=0 2024-11-28T06:54:05,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741907_1083 (size=12301) 2024-11-28T06:54:05,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741908_1084 (size=13051) 2024-11-28T06:54:05,222 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/eeed1ec3f89243b3a1b5d6b46a54fee4 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/eeed1ec3f89243b3a1b5d6b46a54fee4 2024-11-28T06:54:05,229 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ca3b1ced864db1c50bb8f904a838f3/C of d6ca3b1ced864db1c50bb8f904a838f3 into eeed1ec3f89243b3a1b5d6b46a54fee4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:05,230 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:54:05,230 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., storeName=d6ca3b1ced864db1c50bb8f904a838f3/C, priority=13, startTime=1732776845143; duration=0sec 2024-11-28T06:54:05,230 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:05,230 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ca3b1ced864db1c50bb8f904a838f3:C 2024-11-28T06:54:05,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-28T06:54:05,575 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/bcdabece3c6440afb7ffaea818e5707e as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/bcdabece3c6440afb7ffaea818e5707e 2024-11-28T06:54:05,584 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ca3b1ced864db1c50bb8f904a838f3/B of d6ca3b1ced864db1c50bb8f904a838f3 into bcdabece3c6440afb7ffaea818e5707e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:05,584 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:54:05,585 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., storeName=d6ca3b1ced864db1c50bb8f904a838f3/B, priority=13, startTime=1732776845143; duration=0sec 2024-11-28T06:54:05,585 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:05,585 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ca3b1ced864db1c50bb8f904a838f3:B 2024-11-28T06:54:05,612 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/a488ef25bf78450098e75eace8093d36 2024-11-28T06:54:05,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/84030f622ab9431eb8f56eb3a153de12 is 50, key is test_row_0/B:col10/1732776843875/Put/seqid=0 2024-11-28T06:54:05,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741909_1085 (size=12301) 2024-11-28T06:54:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:54:06,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:54:06,068 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/84030f622ab9431eb8f56eb3a153de12 2024-11-28T06:54:06,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/3b68886cc2d84aca8c0a9de918f02ed7 is 50, key is test_row_0/C:col10/1732776843875/Put/seqid=0 2024-11-28T06:54:06,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776906078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:06,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:06,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776906080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:06,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741910_1086 (size=12301) 2024-11-28T06:54:06,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776906082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:06,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776906082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:06,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,094 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:06,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:06,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776906185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:06,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:06,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776906196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:06,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:06,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776906196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:06,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:06,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776906196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:06,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-28T06:54:06,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776906396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:06,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:06,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776906405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:06,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:06,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776906405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:06,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:06,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776906406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:06,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:06,492 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/3b68886cc2d84aca8c0a9de918f02ed7 2024-11-28T06:54:06,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/a488ef25bf78450098e75eace8093d36 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/a488ef25bf78450098e75eace8093d36 2024-11-28T06:54:06,508 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/a488ef25bf78450098e75eace8093d36, entries=150, sequenceid=324, filesize=12.0 K 2024-11-28T06:54:06,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/84030f622ab9431eb8f56eb3a153de12 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/84030f622ab9431eb8f56eb3a153de12 2024-11-28T06:54:06,519 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/84030f622ab9431eb8f56eb3a153de12, entries=150, sequenceid=324, filesize=12.0 K 2024-11-28T06:54:06,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/3b68886cc2d84aca8c0a9de918f02ed7 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/3b68886cc2d84aca8c0a9de918f02ed7 2024-11-28T06:54:06,527 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/3b68886cc2d84aca8c0a9de918f02ed7, entries=150, sequenceid=324, filesize=12.0 K 2024-11-28T06:54:06,528 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for d6ca3b1ced864db1c50bb8f904a838f3 in 1339ms, sequenceid=324, compaction requested=false 2024-11-28T06:54:06,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:54:06,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:06,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-28T06:54:06,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-28T06:54:06,534 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-28T06:54:06,534 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4200 sec 2024-11-28T06:54:06,537 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.4280 sec 2024-11-28T06:54:06,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:54:06,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6ca3b1ced864db1c50bb8f904a838f3 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T06:54:06,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=A 2024-11-28T06:54:06,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:06,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=B 2024-11-28T06:54:06,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:06,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=C 2024-11-28T06:54:06,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:06,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/36762199ced842da9c739e66bfdd2dc9 is 50, key is test_row_0/A:col10/1732776846078/Put/seqid=0 2024-11-28T06:54:06,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741911_1087 (size=14741) 2024-11-28T06:54:06,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/36762199ced842da9c739e66bfdd2dc9 2024-11-28T06:54:06,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/2177f68ad3644f9abffab236914e8a30 is 50, key is test_row_0/B:col10/1732776846078/Put/seqid=0 2024-11-28T06:54:06,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741912_1088 (size=12301) 2024-11-28T06:54:06,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/2177f68ad3644f9abffab236914e8a30 2024-11-28T06:54:06,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:06,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776906735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:06,744 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,744 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:06,744 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:06,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776906735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:06,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:06,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776906736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:06,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:06,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:06,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776906737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:06,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:06,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:06,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/dd099dabd4ec4d329bc0e9e44266bf85 is 50, key is test_row_0/C:col10/1732776846078/Put/seqid=0 2024-11-28T06:54:06,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741913_1089 (size=12301) 2024-11-28T06:54:06,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/dd099dabd4ec4d329bc0e9e44266bf85 2024-11-28T06:54:06,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/36762199ced842da9c739e66bfdd2dc9 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/36762199ced842da9c739e66bfdd2dc9 2024-11-28T06:54:06,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/36762199ced842da9c739e66bfdd2dc9, entries=200, sequenceid=344, filesize=14.4 K 2024-11-28T06:54:06,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/2177f68ad3644f9abffab236914e8a30 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/2177f68ad3644f9abffab236914e8a30 2024-11-28T06:54:06,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/2177f68ad3644f9abffab236914e8a30, entries=150, sequenceid=344, filesize=12.0 K 2024-11-28T06:54:06,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/dd099dabd4ec4d329bc0e9e44266bf85 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/dd099dabd4ec4d329bc0e9e44266bf85 2024-11-28T06:54:06,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/dd099dabd4ec4d329bc0e9e44266bf85, entries=150, sequenceid=344, filesize=12.0 K 2024-11-28T06:54:06,818 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for d6ca3b1ced864db1c50bb8f904a838f3 in 108ms, sequenceid=344, compaction requested=true 2024-11-28T06:54:06,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:54:06,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ca3b1ced864db1c50bb8f904a838f3:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:54:06,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:06,819 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:54:06,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ca3b1ced864db1c50bb8f904a838f3:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:54:06,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:06,819 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:54:06,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ca3b1ced864db1c50bb8f904a838f3:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:54:06,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:54:06,820 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:54:06,820 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): d6ca3b1ced864db1c50bb8f904a838f3/A is initiating minor compaction (all files) 2024-11-28T06:54:06,820 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ca3b1ced864db1c50bb8f904a838f3/A in TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:06,820 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/3445a1f8c1d948838ac0f2e743201542, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/a488ef25bf78450098e75eace8093d36, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/36762199ced842da9c739e66bfdd2dc9] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp, totalSize=39.2 K 2024-11-28T06:54:06,821 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:54:06,821 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): d6ca3b1ced864db1c50bb8f904a838f3/B is initiating minor compaction (all files) 2024-11-28T06:54:06,821 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ca3b1ced864db1c50bb8f904a838f3/B in TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:06,821 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/bcdabece3c6440afb7ffaea818e5707e, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/84030f622ab9431eb8f56eb3a153de12, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/2177f68ad3644f9abffab236914e8a30] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp, totalSize=36.8 K 2024-11-28T06:54:06,823 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3445a1f8c1d948838ac0f2e743201542, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732776843745 2024-11-28T06:54:06,823 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting bcdabece3c6440afb7ffaea818e5707e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732776843745 2024-11-28T06:54:06,823 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 84030f622ab9431eb8f56eb3a153de12, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732776843875 2024-11-28T06:54:06,823 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting a488ef25bf78450098e75eace8093d36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732776843875 2024-11-28T06:54:06,824 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 2177f68ad3644f9abffab236914e8a30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732776846078 2024-11-28T06:54:06,824 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36762199ced842da9c739e66bfdd2dc9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732776846078 2024-11-28T06:54:06,836 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ca3b1ced864db1c50bb8f904a838f3#B#compaction#77 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:54:06,836 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ca3b1ced864db1c50bb8f904a838f3#A#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:54:06,837 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/4ffaf290ff3c476fac72267c6382da0c is 50, key is test_row_0/B:col10/1732776846078/Put/seqid=0 2024-11-28T06:54:06,837 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/95a65eaee9014cabb1121f33530f522d is 50, key is test_row_0/A:col10/1732776846078/Put/seqid=0 2024-11-28T06:54:06,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741914_1090 (size=13187) 2024-11-28T06:54:06,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741915_1091 (size=13187) 2024-11-28T06:54:06,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:54:06,862 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6ca3b1ced864db1c50bb8f904a838f3 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-28T06:54:06,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=A 2024-11-28T06:54:06,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:06,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=B 2024-11-28T06:54:06,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:06,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=C 2024-11-28T06:54:06,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:06,867 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/95a65eaee9014cabb1121f33530f522d as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/95a65eaee9014cabb1121f33530f522d 2024-11-28T06:54:06,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/a21dece0323946528d5a0b9cd672a1ce is 50, key is test_row_0/A:col10/1732776846726/Put/seqid=0 2024-11-28T06:54:06,880 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ca3b1ced864db1c50bb8f904a838f3/A of d6ca3b1ced864db1c50bb8f904a838f3 into 95a65eaee9014cabb1121f33530f522d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:06,880 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:54:06,880 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., storeName=d6ca3b1ced864db1c50bb8f904a838f3/A, priority=13, startTime=1732776846819; duration=0sec 2024-11-28T06:54:06,880 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:54:06,880 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ca3b1ced864db1c50bb8f904a838f3:A 2024-11-28T06:54:06,880 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:54:06,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741916_1092 (size=12301) 2024-11-28T06:54:06,892 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:54:06,892 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): d6ca3b1ced864db1c50bb8f904a838f3/C is initiating minor compaction (all files) 2024-11-28T06:54:06,892 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ca3b1ced864db1c50bb8f904a838f3/C in TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:06,892 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/eeed1ec3f89243b3a1b5d6b46a54fee4, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/3b68886cc2d84aca8c0a9de918f02ed7, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/dd099dabd4ec4d329bc0e9e44266bf85] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp, totalSize=36.8 K 2024-11-28T06:54:06,892 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting eeed1ec3f89243b3a1b5d6b46a54fee4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732776843745 2024-11-28T06:54:06,893 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b68886cc2d84aca8c0a9de918f02ed7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732776843875 2024-11-28T06:54:06,893 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd099dabd4ec4d329bc0e9e44266bf85, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732776846078 2024-11-28T06:54:06,903 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ca3b1ced864db1c50bb8f904a838f3#C#compaction#80 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:54:06,903 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/c9a9fba8fc8440a18988b4e708b1ee53 is 50, key is test_row_0/C:col10/1732776846078/Put/seqid=0 2024-11-28T06:54:06,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741917_1093 (size=13153) 2024-11-28T06:54:06,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:06,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776906931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:06,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:06,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:06,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776906931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:06,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:06,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:06,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776906932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:06,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:06,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776906940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:06,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:06,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:07,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776907046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:07,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:07,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:07,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:07,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776907046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:07,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:07,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:07,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:07,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:07,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776907046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:07,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:07,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:07,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:07,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776907056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:07,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:07,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:07,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:07,250 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/4ffaf290ff3c476fac72267c6382da0c as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/4ffaf290ff3c476fac72267c6382da0c 2024-11-28T06:54:07,257 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ca3b1ced864db1c50bb8f904a838f3/B of d6ca3b1ced864db1c50bb8f904a838f3 into 4ffaf290ff3c476fac72267c6382da0c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:07,257 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:54:07,257 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., storeName=d6ca3b1ced864db1c50bb8f904a838f3/B, priority=13, startTime=1732776846819; duration=0sec 2024-11-28T06:54:07,257 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:07,257 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ca3b1ced864db1c50bb8f904a838f3:B 2024-11-28T06:54:07,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776907266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:07,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776907266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:07,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:07,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:07,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:07,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:07,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776907267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:07,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:07,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:07,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:07,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:07,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776907267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:07,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/a21dece0323946528d5a0b9cd672a1ce 2024-11-28T06:54:07,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/6d6719bcd7f9414f9c5d155b09c0a332 is 50, key is test_row_0/B:col10/1732776846726/Put/seqid=0 2024-11-28T06:54:07,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741918_1094 (size=12301) 2024-11-28T06:54:07,320 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/6d6719bcd7f9414f9c5d155b09c0a332 2024-11-28T06:54:07,320 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/c9a9fba8fc8440a18988b4e708b1ee53 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/c9a9fba8fc8440a18988b4e708b1ee53 2024-11-28T06:54:07,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/38ac7a2a88864e4eaa1907599c63db44 is 50, key is test_row_0/C:col10/1732776846726/Put/seqid=0 2024-11-28T06:54:07,332 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ca3b1ced864db1c50bb8f904a838f3/C of d6ca3b1ced864db1c50bb8f904a838f3 into c9a9fba8fc8440a18988b4e708b1ee53(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:07,332 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:54:07,332 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., storeName=d6ca3b1ced864db1c50bb8f904a838f3/C, priority=13, startTime=1732776846819; duration=0sec 2024-11-28T06:54:07,332 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:07,332 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ca3b1ced864db1c50bb8f904a838f3:C 2024-11-28T06:54:07,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741919_1095 (size=12301) 2024-11-28T06:54:07,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:07,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776907575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:07,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:07,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:07,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:07,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:07,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776907576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:07,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:07,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:07,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:07,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:07,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776907585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:07,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:07,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:07,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:07,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:07,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776907585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:07,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:07,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:07,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:07,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/38ac7a2a88864e4eaa1907599c63db44 2024-11-28T06:54:07,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/a21dece0323946528d5a0b9cd672a1ce as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/a21dece0323946528d5a0b9cd672a1ce 2024-11-28T06:54:07,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/a21dece0323946528d5a0b9cd672a1ce, entries=150, sequenceid=360, filesize=12.0 K 2024-11-28T06:54:07,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/6d6719bcd7f9414f9c5d155b09c0a332 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/6d6719bcd7f9414f9c5d155b09c0a332 2024-11-28T06:54:07,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/6d6719bcd7f9414f9c5d155b09c0a332, entries=150, sequenceid=360, filesize=12.0 K 2024-11-28T06:54:07,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/38ac7a2a88864e4eaa1907599c63db44 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/38ac7a2a88864e4eaa1907599c63db44 2024-11-28T06:54:07,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/38ac7a2a88864e4eaa1907599c63db44, entries=150, sequenceid=360, filesize=12.0 K 2024-11-28T06:54:07,771 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for d6ca3b1ced864db1c50bb8f904a838f3 in 909ms, sequenceid=360, compaction requested=false 2024-11-28T06:54:07,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:54:08,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:54:08,088 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6ca3b1ced864db1c50bb8f904a838f3 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-28T06:54:08,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=A 2024-11-28T06:54:08,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:08,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=B 2024-11-28T06:54:08,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:08,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=C 2024-11-28T06:54:08,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:08,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/9d3c459e37d344a481bb5fc10e583414 is 50, key is test_row_0/A:col10/1732776846894/Put/seqid=0 2024-11-28T06:54:08,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741920_1096 (size=14741) 2024-11-28T06:54:08,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:08,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776908105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:08,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:08,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:08,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:08,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:08,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:08,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776908113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:08,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776908112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:08,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:08,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:08,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:08,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776908116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:08,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:08,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:08,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:08,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:08,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:08,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:08,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:08,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:08,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776908225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:08,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:08,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:08,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:08,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:08,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776908225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:08,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:08,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:08,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:08,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:08,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776908225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:08,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:08,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:08,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:08,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:08,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776908226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:08,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:08,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:08,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:08,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-28T06:54:08,275 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:54:08,276 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:54:08,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-28T06:54:08,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-28T06:54:08,278 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:54:08,279 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:54:08,279 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:54:08,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-28T06:54:08,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-28T06:54:08,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:08,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:54:08,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:08,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:08,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:08,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:08,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:08,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776908435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:08,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:08,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:08,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:08,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:08,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776908435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:08,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:08,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:08,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:08,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:08,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776908436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:08,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:08,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:08,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776908436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:08,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:08,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:08,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:08,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:08,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:08,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/9d3c459e37d344a481bb5fc10e583414 2024-11-28T06:54:08,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/72d05257f24e42c7bb22f64e7ea8c97a is 50, key is test_row_0/B:col10/1732776846894/Put/seqid=0 2024-11-28T06:54:08,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741921_1097 (size=12301) 2024-11-28T06:54:08,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-28T06:54:08,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:08,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:54:08,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:08,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:08,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:08,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:08,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-28T06:54:08,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-28T06:54:08,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:08,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:54:08,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:08,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:08,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:08,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:08,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:08,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776908745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:08,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:08,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:08,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:08,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:08,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776908756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:08,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:08,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776908756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:08,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:08,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:08,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:08,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:08,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776908756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:08,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:08,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:08,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:08,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:08,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:08,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:08,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-28T06:54:08,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:08,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:54:08,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:08,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:08,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:08,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:08,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-28T06:54:08,917 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/72d05257f24e42c7bb22f64e7ea8c97a 2024-11-28T06:54:08,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/f8eb624f370345318aabb8a8fc34fda5 is 50, key is test_row_0/C:col10/1732776846894/Put/seqid=0 2024-11-28T06:54:08,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741922_1098 (size=12301) 2024-11-28T06:54:09,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-28T06:54:09,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:09,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:54:09,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:09,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:09,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:09,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-28T06:54:09,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:09,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:54:09,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:09,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:09,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:09,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:09,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:09,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776909265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:09,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:09,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:09,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:09,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:09,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776909265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:09,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:09,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:09,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776909265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:09,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:09,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:09,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:09,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:09,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:09,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776909265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:09,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:09,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:09,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:09,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:09,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/f8eb624f370345318aabb8a8fc34fda5 2024-11-28T06:54:09,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/9d3c459e37d344a481bb5fc10e583414 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/9d3c459e37d344a481bb5fc10e583414 2024-11-28T06:54:09,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/9d3c459e37d344a481bb5fc10e583414, entries=200, sequenceid=380, filesize=14.4 K 2024-11-28T06:54:09,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/72d05257f24e42c7bb22f64e7ea8c97a as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/72d05257f24e42c7bb22f64e7ea8c97a 2024-11-28T06:54:09,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-28T06:54:09,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:09,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:54:09,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:09,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:09,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:09,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/72d05257f24e42c7bb22f64e7ea8c97a, entries=150, sequenceid=380, filesize=12.0 K 2024-11-28T06:54:09,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/f8eb624f370345318aabb8a8fc34fda5 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/f8eb624f370345318aabb8a8fc34fda5 2024-11-28T06:54:09,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/f8eb624f370345318aabb8a8fc34fda5, entries=150, sequenceid=380, filesize=12.0 K 2024-11-28T06:54:09,362 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for d6ca3b1ced864db1c50bb8f904a838f3 in 1275ms, sequenceid=380, compaction requested=true 2024-11-28T06:54:09,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:54:09,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ca3b1ced864db1c50bb8f904a838f3:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:54:09,362 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:54:09,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:09,362 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:54:09,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ca3b1ced864db1c50bb8f904a838f3:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:54:09,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:09,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ca3b1ced864db1c50bb8f904a838f3:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:54:09,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:54:09,364 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:54:09,364 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:54:09,364 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): d6ca3b1ced864db1c50bb8f904a838f3/B is initiating minor compaction (all files) 2024-11-28T06:54:09,364 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): d6ca3b1ced864db1c50bb8f904a838f3/A is initiating minor compaction (all files) 2024-11-28T06:54:09,364 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ca3b1ced864db1c50bb8f904a838f3/B in TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:09,364 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ca3b1ced864db1c50bb8f904a838f3/A in TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:09,364 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/4ffaf290ff3c476fac72267c6382da0c, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/6d6719bcd7f9414f9c5d155b09c0a332, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/72d05257f24e42c7bb22f64e7ea8c97a] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp, totalSize=36.9 K 2024-11-28T06:54:09,364 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/95a65eaee9014cabb1121f33530f522d, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/a21dece0323946528d5a0b9cd672a1ce, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/9d3c459e37d344a481bb5fc10e583414] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp, totalSize=39.3 K 2024-11-28T06:54:09,365 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ffaf290ff3c476fac72267c6382da0c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732776846078 2024-11-28T06:54:09,365 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95a65eaee9014cabb1121f33530f522d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732776846078 2024-11-28T06:54:09,365 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting a21dece0323946528d5a0b9cd672a1ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732776846726 2024-11-28T06:54:09,365 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d6719bcd7f9414f9c5d155b09c0a332, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732776846726 2024-11-28T06:54:09,366 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d3c459e37d344a481bb5fc10e583414, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732776846884 2024-11-28T06:54:09,366 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 72d05257f24e42c7bb22f64e7ea8c97a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732776846884 2024-11-28T06:54:09,375 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ca3b1ced864db1c50bb8f904a838f3#B#compaction#86 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:54:09,375 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/fce0dbe5289d4bc49f7893241dce974d is 50, key is test_row_0/B:col10/1732776846894/Put/seqid=0 2024-11-28T06:54:09,385 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ca3b1ced864db1c50bb8f904a838f3#A#compaction#87 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:54:09,386 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/366541860f1c4026863a3f5032e56e72 is 50, key is test_row_0/A:col10/1732776846894/Put/seqid=0 2024-11-28T06:54:09,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741923_1099 (size=13289) 2024-11-28T06:54:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-28T06:54:09,423 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/fce0dbe5289d4bc49f7893241dce974d as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/fce0dbe5289d4bc49f7893241dce974d 2024-11-28T06:54:09,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741924_1100 (size=13289) 2024-11-28T06:54:09,430 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ca3b1ced864db1c50bb8f904a838f3/B of d6ca3b1ced864db1c50bb8f904a838f3 into fce0dbe5289d4bc49f7893241dce974d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:09,430 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:54:09,430 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., storeName=d6ca3b1ced864db1c50bb8f904a838f3/B, priority=13, startTime=1732776849362; duration=0sec 2024-11-28T06:54:09,430 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:54:09,430 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ca3b1ced864db1c50bb8f904a838f3:B 2024-11-28T06:54:09,431 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:54:09,432 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:54:09,432 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): d6ca3b1ced864db1c50bb8f904a838f3/C is initiating minor compaction (all files) 2024-11-28T06:54:09,432 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ca3b1ced864db1c50bb8f904a838f3/C in TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:09,432 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/c9a9fba8fc8440a18988b4e708b1ee53, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/38ac7a2a88864e4eaa1907599c63db44, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/f8eb624f370345318aabb8a8fc34fda5] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp, totalSize=36.9 K 2024-11-28T06:54:09,433 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting c9a9fba8fc8440a18988b4e708b1ee53, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732776846078 2024-11-28T06:54:09,433 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 38ac7a2a88864e4eaa1907599c63db44, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732776846726 2024-11-28T06:54:09,433 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting f8eb624f370345318aabb8a8fc34fda5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732776846884 2024-11-28T06:54:09,441 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ca3b1ced864db1c50bb8f904a838f3#C#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:54:09,442 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/f3fcd0b4262d4dddbe6994e8eb656f13 is 50, key is test_row_0/C:col10/1732776846894/Put/seqid=0 2024-11-28T06:54:09,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741925_1101 (size=13255) 2024-11-28T06:54:09,456 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/f3fcd0b4262d4dddbe6994e8eb656f13 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/f3fcd0b4262d4dddbe6994e8eb656f13 2024-11-28T06:54:09,462 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ca3b1ced864db1c50bb8f904a838f3/C of d6ca3b1ced864db1c50bb8f904a838f3 into f3fcd0b4262d4dddbe6994e8eb656f13(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:09,462 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:54:09,462 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., storeName=d6ca3b1ced864db1c50bb8f904a838f3/C, priority=13, startTime=1732776849362; duration=0sec 2024-11-28T06:54:09,462 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:09,462 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ca3b1ced864db1c50bb8f904a838f3:C 2024-11-28T06:54:09,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-28T06:54:09,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:09,505 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing d6ca3b1ced864db1c50bb8f904a838f3 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-28T06:54:09,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=A 2024-11-28T06:54:09,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:09,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=B 2024-11-28T06:54:09,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:09,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=C 2024-11-28T06:54:09,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:09,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/6dd1cacc7c45461790141f41e9c8c48f is 50, key is test_row_0/A:col10/1732776848111/Put/seqid=0 2024-11-28T06:54:09,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741926_1102 (size=12301) 2024-11-28T06:54:09,829 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/366541860f1c4026863a3f5032e56e72 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/366541860f1c4026863a3f5032e56e72 2024-11-28T06:54:09,839 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ca3b1ced864db1c50bb8f904a838f3/A of d6ca3b1ced864db1c50bb8f904a838f3 into 366541860f1c4026863a3f5032e56e72(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:09,839 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:54:09,840 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., storeName=d6ca3b1ced864db1c50bb8f904a838f3/A, priority=13, startTime=1732776849362; duration=0sec 2024-11-28T06:54:09,840 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:09,840 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ca3b1ced864db1c50bb8f904a838f3:A 2024-11-28T06:54:09,914 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/6dd1cacc7c45461790141f41e9c8c48f 2024-11-28T06:54:09,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/bf0806a51017421fafedc89966e54e19 is 50, key is test_row_0/B:col10/1732776848111/Put/seqid=0 2024-11-28T06:54:09,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741927_1103 (size=12301) 2024-11-28T06:54:10,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:54:10,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. as already flushing 2024-11-28T06:54:10,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:10,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36870 deadline: 1732776910115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:10,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:10,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:10,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:10,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:10,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36870 deadline: 1732776910225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:10,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:10,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:10,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:10,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:10,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776910276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:10,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:10,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776910276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:10,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:10,280 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_1' in d6ca3b1ced864db1c50bb8f904a838f3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:54:10,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776910276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:10,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:10,280 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_0' in d6ca3b1ced864db1c50bb8f904a838f3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:54:10,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:10,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:10,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:10,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:10,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:10,283 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_0' in d6ca3b1ced864db1c50bb8f904a838f3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:54:10,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:10,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:10,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:10,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:10,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776910285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:10,286 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_0' in d6ca3b1ced864db1c50bb8f904a838f3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:54:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:10,331 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/bf0806a51017421fafedc89966e54e19 2024-11-28T06:54:10,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/e604bd7d7620476cb7a193946c482a13 is 50, key is test_row_0/C:col10/1732776848111/Put/seqid=0 2024-11-28T06:54:10,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741928_1104 (size=12301) 2024-11-28T06:54:10,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-28T06:54:10,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:10,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36870 deadline: 1732776910436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:10,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:10,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:10,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:10,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:10,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36870 deadline: 1732776910745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:10,754 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/e604bd7d7620476cb7a193946c482a13 2024-11-28T06:54:10,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/6dd1cacc7c45461790141f41e9c8c48f as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/6dd1cacc7c45461790141f41e9c8c48f 2024-11-28T06:54:10,772 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/6dd1cacc7c45461790141f41e9c8c48f, entries=150, sequenceid=398, filesize=12.0 K 2024-11-28T06:54:10,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/bf0806a51017421fafedc89966e54e19 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/bf0806a51017421fafedc89966e54e19 2024-11-28T06:54:10,782 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/bf0806a51017421fafedc89966e54e19, entries=150, sequenceid=398, filesize=12.0 K 2024-11-28T06:54:10,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/e604bd7d7620476cb7a193946c482a13 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/e604bd7d7620476cb7a193946c482a13 2024-11-28T06:54:10,791 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/e604bd7d7620476cb7a193946c482a13, entries=150, sequenceid=398, filesize=12.0 K 2024-11-28T06:54:10,793 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for d6ca3b1ced864db1c50bb8f904a838f3 in 1288ms, sequenceid=398, compaction requested=false 2024-11-28T06:54:10,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:54:10,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:10,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-28T06:54:10,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-28T06:54:10,797 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-28T06:54:10,797 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5160 sec 2024-11-28T06:54:10,801 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.5220 sec 2024-11-28T06:54:11,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:54:11,259 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6ca3b1ced864db1c50bb8f904a838f3 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-28T06:54:11,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=A 2024-11-28T06:54:11,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:11,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=B 2024-11-28T06:54:11,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:11,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=C 2024-11-28T06:54:11,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:11,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/e5a48f79fe0040548e0babd968615db0 is 50, key is test_row_0/A:col10/1732776850113/Put/seqid=0 2024-11-28T06:54:11,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741929_1105 (size=14741) 2024-11-28T06:54:11,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36870 deadline: 1732776911320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:11,326 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:11,326 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:11,326 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:11,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:11,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36870 deadline: 1732776911436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:11,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:11,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:11,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:11,445 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-28T06:54:11,446 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:54:11,450 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-28T06:54:11,451 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:54:11,453 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-28T06:54:11,453 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:54:11,453 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:11,453 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:11,454 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:11,454 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:11,454 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:54:11,455 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:11,455 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:11,455 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:54:11,455 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-28T06:54:11,455 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:54:11,455 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:11,456 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:11,456 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:54:11,456 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:54:11,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:11,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36870 deadline: 1732776911646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:11,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:11,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:11,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:11,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/e5a48f79fe0040548e0babd968615db0 2024-11-28T06:54:11,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/97040d32c6054fbaab43606128f8a882 is 50, key is test_row_0/B:col10/1732776850113/Put/seqid=0 2024-11-28T06:54:11,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741930_1106 (size=12301) 2024-11-28T06:54:11,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:11,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36870 deadline: 1732776911956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:11,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:11,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:11,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:12,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/97040d32c6054fbaab43606128f8a882 2024-11-28T06:54:12,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/ca8adaf6464844a4ad873907e87278a0 is 50, key is test_row_0/C:col10/1732776850113/Put/seqid=0 2024-11-28T06:54:12,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741931_1107 (size=12301) 2024-11-28T06:54:12,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:12,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36946 deadline: 1732776912296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36884 deadline: 1732776912296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:12,299 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_0' in d6ca3b1ced864db1c50bb8f904a838f3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:54:12,299 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_0' in d6ca3b1ced864db1c50bb8f904a838f3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:54:12,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:12,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:12,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:12,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:12,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:12,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:12,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:12,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1732776912305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:12,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:12,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36856 deadline: 1732776912306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:12,306 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_1' in d6ca3b1ced864db1c50bb8f904a838f3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:54:12,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:12,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:12,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:12,307 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_0' in d6ca3b1ced864db1c50bb8f904a838f3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:54:12,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:12,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:12,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:12,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-28T06:54:12,436 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:54:12,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36870 deadline: 1732776912465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:12,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:12,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:12,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:12,533 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/ca8adaf6464844a4ad873907e87278a0 2024-11-28T06:54:12,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/e5a48f79fe0040548e0babd968615db0 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/e5a48f79fe0040548e0babd968615db0 2024-11-28T06:54:12,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/e5a48f79fe0040548e0babd968615db0, entries=200, sequenceid=416, filesize=14.4 K 2024-11-28T06:54:12,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/97040d32c6054fbaab43606128f8a882 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/97040d32c6054fbaab43606128f8a882 2024-11-28T06:54:12,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/97040d32c6054fbaab43606128f8a882, entries=150, sequenceid=416, filesize=12.0 K 2024-11-28T06:54:12,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/ca8adaf6464844a4ad873907e87278a0 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/ca8adaf6464844a4ad873907e87278a0 2024-11-28T06:54:12,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/ca8adaf6464844a4ad873907e87278a0, entries=150, sequenceid=416, filesize=12.0 K 2024-11-28T06:54:12,563 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for d6ca3b1ced864db1c50bb8f904a838f3 in 1304ms, sequenceid=416, compaction requested=true 2024-11-28T06:54:12,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:54:12,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ca3b1ced864db1c50bb8f904a838f3:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:54:12,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:12,564 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:54:12,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ca3b1ced864db1c50bb8f904a838f3:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:54:12,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:12,564 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:54:12,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6ca3b1ced864db1c50bb8f904a838f3:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:54:12,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:54:12,565 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40331 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:54:12,565 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:54:12,565 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): d6ca3b1ced864db1c50bb8f904a838f3/B is initiating minor compaction (all files) 2024-11-28T06:54:12,565 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): d6ca3b1ced864db1c50bb8f904a838f3/A is initiating minor compaction (all files) 2024-11-28T06:54:12,565 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ca3b1ced864db1c50bb8f904a838f3/A in TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:12,565 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ca3b1ced864db1c50bb8f904a838f3/B in TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:12,565 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/366541860f1c4026863a3f5032e56e72, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/6dd1cacc7c45461790141f41e9c8c48f, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/e5a48f79fe0040548e0babd968615db0] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp, totalSize=39.4 K 2024-11-28T06:54:12,565 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/fce0dbe5289d4bc49f7893241dce974d, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/bf0806a51017421fafedc89966e54e19, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/97040d32c6054fbaab43606128f8a882] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp, totalSize=37.0 K 2024-11-28T06:54:12,566 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 366541860f1c4026863a3f5032e56e72, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732776846884 2024-11-28T06:54:12,566 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting fce0dbe5289d4bc49f7893241dce974d, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732776846884 2024-11-28T06:54:12,566 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6dd1cacc7c45461790141f41e9c8c48f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732776848098 2024-11-28T06:54:12,566 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting bf0806a51017421fafedc89966e54e19, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732776848098 2024-11-28T06:54:12,566 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5a48f79fe0040548e0babd968615db0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732776850096 2024-11-28T06:54:12,566 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 97040d32c6054fbaab43606128f8a882, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732776850096 2024-11-28T06:54:12,575 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ca3b1ced864db1c50bb8f904a838f3#B#compaction#95 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:54:12,575 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/810896d6d90149cab31fe06e54273914 is 50, key is test_row_0/B:col10/1732776850113/Put/seqid=0 2024-11-28T06:54:12,577 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ca3b1ced864db1c50bb8f904a838f3#A#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:54:12,578 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/fc5c8b149cef4b66bd78561abf5bf83d is 50, key is test_row_0/A:col10/1732776850113/Put/seqid=0 2024-11-28T06:54:12,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741932_1108 (size=13391) 2024-11-28T06:54:12,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741933_1109 (size=13391) 2024-11-28T06:54:12,990 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/810896d6d90149cab31fe06e54273914 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/810896d6d90149cab31fe06e54273914 2024-11-28T06:54:12,992 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/fc5c8b149cef4b66bd78561abf5bf83d as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/fc5c8b149cef4b66bd78561abf5bf83d 2024-11-28T06:54:12,998 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ca3b1ced864db1c50bb8f904a838f3/B of d6ca3b1ced864db1c50bb8f904a838f3 into 810896d6d90149cab31fe06e54273914(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:12,998 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ca3b1ced864db1c50bb8f904a838f3/A of d6ca3b1ced864db1c50bb8f904a838f3 into fc5c8b149cef4b66bd78561abf5bf83d(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:12,998 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:54:12,998 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:54:12,998 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., storeName=d6ca3b1ced864db1c50bb8f904a838f3/A, priority=13, startTime=1732776852563; duration=0sec 2024-11-28T06:54:12,998 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., storeName=d6ca3b1ced864db1c50bb8f904a838f3/B, priority=13, startTime=1732776852564; duration=0sec 2024-11-28T06:54:12,998 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:54:12,998 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ca3b1ced864db1c50bb8f904a838f3:B 2024-11-28T06:54:12,998 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:54:12,998 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ca3b1ced864db1c50bb8f904a838f3:A 2024-11-28T06:54:12,998 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:54:12,999 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:54:12,999 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): d6ca3b1ced864db1c50bb8f904a838f3/C is initiating minor compaction (all files) 2024-11-28T06:54:12,999 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6ca3b1ced864db1c50bb8f904a838f3/C in TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:13,000 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/f3fcd0b4262d4dddbe6994e8eb656f13, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/e604bd7d7620476cb7a193946c482a13, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/ca8adaf6464844a4ad873907e87278a0] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp, totalSize=37.0 K 2024-11-28T06:54:13,000 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting f3fcd0b4262d4dddbe6994e8eb656f13, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732776846884 2024-11-28T06:54:13,000 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting e604bd7d7620476cb7a193946c482a13, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732776848098 2024-11-28T06:54:13,000 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting ca8adaf6464844a4ad873907e87278a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732776850096 2024-11-28T06:54:13,008 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6ca3b1ced864db1c50bb8f904a838f3#C#compaction#97 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:54:13,008 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/0cbd0512d5d841d7a8cfee2ef1531339 is 50, key is test_row_0/C:col10/1732776850113/Put/seqid=0 2024-11-28T06:54:13,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741934_1110 (size=13357) 2024-11-28T06:54:13,425 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/0cbd0512d5d841d7a8cfee2ef1531339 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/0cbd0512d5d841d7a8cfee2ef1531339 2024-11-28T06:54:13,434 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6ca3b1ced864db1c50bb8f904a838f3/C of d6ca3b1ced864db1c50bb8f904a838f3 into 0cbd0512d5d841d7a8cfee2ef1531339(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:13,434 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:54:13,434 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., storeName=d6ca3b1ced864db1c50bb8f904a838f3/C, priority=13, startTime=1732776852564; duration=0sec 2024-11-28T06:54:13,434 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:13,435 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6ca3b1ced864db1c50bb8f904a838f3:C 2024-11-28T06:54:13,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:54:13,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6ca3b1ced864db1c50bb8f904a838f3 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-28T06:54:13,477 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-28T06:54:13,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=A 2024-11-28T06:54:13,477 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:54:13,477 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:13,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:13,477 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:13,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=B 2024-11-28T06:54:13,477 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:54:13,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:13,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=C 2024-11-28T06:54:13,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:13,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/fc3193b37937486a8dadebb0df4e4338 is 50, key is test_row_0/A:col10/1732776851302/Put/seqid=0 2024-11-28T06:54:13,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741935_1111 (size=12301) 2024-11-28T06:54:13,868 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T06:54:13,918 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/fc3193b37937486a8dadebb0df4e4338 2024-11-28T06:54:13,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/c486ba7fc94d40b7ab6a88e072b4fe5b is 50, key is test_row_0/B:col10/1732776851302/Put/seqid=0 2024-11-28T06:54:13,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741936_1112 (size=12301) 2024-11-28T06:54:14,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/c486ba7fc94d40b7ab6a88e072b4fe5b 2024-11-28T06:54:14,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/aa0cd0bceea1409fb900f0a0d1450266 is 50, key is test_row_0/C:col10/1732776851302/Put/seqid=0 2024-11-28T06:54:14,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741937_1113 (size=12301) 2024-11-28T06:54:14,745 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/aa0cd0bceea1409fb900f0a0d1450266 2024-11-28T06:54:14,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/fc3193b37937486a8dadebb0df4e4338 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/fc3193b37937486a8dadebb0df4e4338 2024-11-28T06:54:14,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/fc3193b37937486a8dadebb0df4e4338, entries=150, sequenceid=435, filesize=12.0 K 2024-11-28T06:54:14,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/c486ba7fc94d40b7ab6a88e072b4fe5b as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/c486ba7fc94d40b7ab6a88e072b4fe5b 2024-11-28T06:54:14,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/c486ba7fc94d40b7ab6a88e072b4fe5b, entries=150, sequenceid=435, filesize=12.0 K 2024-11-28T06:54:14,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/aa0cd0bceea1409fb900f0a0d1450266 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/aa0cd0bceea1409fb900f0a0d1450266 2024-11-28T06:54:14,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/aa0cd0bceea1409fb900f0a0d1450266, entries=150, sequenceid=435, filesize=12.0 K 2024-11-28T06:54:14,775 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for d6ca3b1ced864db1c50bb8f904a838f3 in 1297ms, sequenceid=435, compaction requested=false 2024-11-28T06:54:14,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6ca3b1ced864db1c50bb8f904a838f3: 2024-11-28T06:54:16,329 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-28T06:54:16,329 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:54:16,329 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:16,329 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:16,329 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:54:16,337 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-28T06:54:16,337 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-28T06:54:16,337 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:54:16,337 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:54:16,337 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:16,337 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:16,337 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:16,337 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:16,337 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:54:16,337 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:54:16,348 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-28T06:54:16,348 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:54:16,348 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:16,348 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:16,348 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:54:16,348 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-28T06:54:16,348 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-28T06:54:16,348 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-28T06:54:16,348 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2024-11-28T06:54:16,349 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-28T06:54:16,349 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-28T06:54:16,349 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-28T06:54:16,349 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2779 2024-11-28T06:54:16,349 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2578 2024-11-28T06:54:16,349 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-28T06:54:16,349 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1183 2024-11-28T06:54:16,349 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3548 rows 2024-11-28T06:54:16,349 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1181 2024-11-28T06:54:16,349 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3542 rows 2024-11-28T06:54:16,349 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T06:54:16,349 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T06:54:16,349 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:16,349 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:16,349 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:54:16,353 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-28T06:54:16,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-28T06:54:16,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-28T06:54:16,360 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732776856360"}]},"ts":"1732776856360"} 2024-11-28T06:54:16,362 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-28T06:54:16,362 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-28T06:54:16,363 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T06:54:16,367 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d6ca3b1ced864db1c50bb8f904a838f3, UNASSIGN}] 2024-11-28T06:54:16,368 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=20, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d6ca3b1ced864db1c50bb8f904a838f3, UNASSIGN 2024-11-28T06:54:16,369 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=d6ca3b1ced864db1c50bb8f904a838f3, regionState=CLOSING, regionLocation=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:16,371 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d6ca3b1ced864db1c50bb8f904a838f3, UNASSIGN because future has completed 2024-11-28T06:54:16,371 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T06:54:16,372 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; CloseRegionProcedure d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568}] 2024-11-28T06:54:16,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-28T06:54:16,531 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] handler.UnassignRegionHandler(122): Close d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:54:16,531 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T06:54:16,532 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1722): Closing d6ca3b1ced864db1c50bb8f904a838f3, disabling compactions & flushes 2024-11-28T06:54:16,532 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:16,532 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:16,532 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. after waiting 0 ms 2024-11-28T06:54:16,532 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:16,532 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(2902): Flushing d6ca3b1ced864db1c50bb8f904a838f3 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-28T06:54:16,533 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=A 2024-11-28T06:54:16,533 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:16,533 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=B 2024-11-28T06:54:16,533 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:16,533 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d6ca3b1ced864db1c50bb8f904a838f3, store=C 2024-11-28T06:54:16,533 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:16,540 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/8d310e22d2c74a80b688edd9e30fc2a3 is 50, key is test_row_0/A:col10/1732776856346/Put/seqid=0 2024-11-28T06:54:16,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741938_1114 (size=9857) 2024-11-28T06:54:16,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-28T06:54:16,945 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/8d310e22d2c74a80b688edd9e30fc2a3 2024-11-28T06:54:16,962 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/43c033f7b76e42c680701ee3029a939f is 50, key is test_row_0/B:col10/1732776856346/Put/seqid=0 2024-11-28T06:54:16,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741939_1115 (size=9857) 2024-11-28T06:54:16,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-28T06:54:17,367 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/43c033f7b76e42c680701ee3029a939f 2024-11-28T06:54:17,378 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/193ec61a1ce84d7b9e1aa5e974982e5c is 50, key is test_row_0/C:col10/1732776856346/Put/seqid=0 2024-11-28T06:54:17,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741940_1116 (size=9857) 2024-11-28T06:54:17,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-28T06:54:17,786 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/193ec61a1ce84d7b9e1aa5e974982e5c 2024-11-28T06:54:17,794 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/A/8d310e22d2c74a80b688edd9e30fc2a3 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/8d310e22d2c74a80b688edd9e30fc2a3 2024-11-28T06:54:17,802 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/8d310e22d2c74a80b688edd9e30fc2a3, entries=100, sequenceid=442, filesize=9.6 K 2024-11-28T06:54:17,803 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/B/43c033f7b76e42c680701ee3029a939f as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/43c033f7b76e42c680701ee3029a939f 2024-11-28T06:54:17,808 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/43c033f7b76e42c680701ee3029a939f, entries=100, sequenceid=442, filesize=9.6 K 2024-11-28T06:54:17,809 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/.tmp/C/193ec61a1ce84d7b9e1aa5e974982e5c as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/193ec61a1ce84d7b9e1aa5e974982e5c 2024-11-28T06:54:17,814 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/193ec61a1ce84d7b9e1aa5e974982e5c, entries=100, sequenceid=442, filesize=9.6 K 2024-11-28T06:54:17,815 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for d6ca3b1ced864db1c50bb8f904a838f3 in 1283ms, sequenceid=442, compaction requested=true 2024-11-28T06:54:17,816 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/40db68802f584dd8afa8de305e7cd21b, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/b50a969913594b0ea4c7126072943c78, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/2cd3cb344ce443059d511a94e031308a, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/2399290eca8d4f2ea87435b1e9f7920e, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/a4c78a863a4f466d9ad433f3959f4f89, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/3d1731be43584894be2ca036799be8d2, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/6d3dcc6500184144a0b5c37d02e6a002, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/07dd5a78d1c34702b36ba234cd327f4e, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/9c6ca5993ec84d8ba241733fdd5c2153, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/6a0e2737f1cc453a984495dd8199368b, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/a9e13f7e074441608fa9c7f6f7e00529, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/dd28d6c98e0d4e9286d6b71539737d42, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/e3b35fd57dca43ce9b95ac510c8c73b0, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/af69f679760c4a4ebd8488592075933e, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/da856ae4fcc8463ea3ded63e13a2eb00, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/5fb72f117b644c2681b18b93ec9fff02, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/2d8f7389c7d8493e87f24bc6ad773dc7, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/5f453a0e67ca4fc08c34db296ba72a97, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/952ec844c1214ce6be5d7f0af5060bb0, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/2dcc9874f8a142f4a8a91ecb5f77a169, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/4da13ca3cd584ecfacb1ea7666e3beb7, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/97cdfe8ad91e40e8b0015b19d6029c00, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/a9ccb262218347de93b04e70f07f6a2f, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/3445a1f8c1d948838ac0f2e743201542, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/a488ef25bf78450098e75eace8093d36, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/36762199ced842da9c739e66bfdd2dc9, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/95a65eaee9014cabb1121f33530f522d, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/a21dece0323946528d5a0b9cd672a1ce, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/9d3c459e37d344a481bb5fc10e583414, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/366541860f1c4026863a3f5032e56e72, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/6dd1cacc7c45461790141f41e9c8c48f, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/e5a48f79fe0040548e0babd968615db0] to archive 2024-11-28T06:54:17,819 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T06:54:17,824 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/40db68802f584dd8afa8de305e7cd21b to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/40db68802f584dd8afa8de305e7cd21b 2024-11-28T06:54:17,826 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/b50a969913594b0ea4c7126072943c78 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/b50a969913594b0ea4c7126072943c78 2024-11-28T06:54:17,828 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/2cd3cb344ce443059d511a94e031308a to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/2cd3cb344ce443059d511a94e031308a 2024-11-28T06:54:17,830 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/2399290eca8d4f2ea87435b1e9f7920e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/2399290eca8d4f2ea87435b1e9f7920e 2024-11-28T06:54:17,832 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/a4c78a863a4f466d9ad433f3959f4f89 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/a4c78a863a4f466d9ad433f3959f4f89 2024-11-28T06:54:17,834 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/3d1731be43584894be2ca036799be8d2 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/3d1731be43584894be2ca036799be8d2 2024-11-28T06:54:17,836 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/6d3dcc6500184144a0b5c37d02e6a002 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/6d3dcc6500184144a0b5c37d02e6a002 2024-11-28T06:54:17,837 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/07dd5a78d1c34702b36ba234cd327f4e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/07dd5a78d1c34702b36ba234cd327f4e 2024-11-28T06:54:17,839 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/9c6ca5993ec84d8ba241733fdd5c2153 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/9c6ca5993ec84d8ba241733fdd5c2153 2024-11-28T06:54:17,840 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/6a0e2737f1cc453a984495dd8199368b to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/6a0e2737f1cc453a984495dd8199368b 2024-11-28T06:54:17,841 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/a9e13f7e074441608fa9c7f6f7e00529 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/a9e13f7e074441608fa9c7f6f7e00529 2024-11-28T06:54:17,842 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/dd28d6c98e0d4e9286d6b71539737d42 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/dd28d6c98e0d4e9286d6b71539737d42 2024-11-28T06:54:17,844 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/e3b35fd57dca43ce9b95ac510c8c73b0 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/e3b35fd57dca43ce9b95ac510c8c73b0 2024-11-28T06:54:17,845 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/af69f679760c4a4ebd8488592075933e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/af69f679760c4a4ebd8488592075933e 2024-11-28T06:54:17,846 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/da856ae4fcc8463ea3ded63e13a2eb00 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/da856ae4fcc8463ea3ded63e13a2eb00 2024-11-28T06:54:17,847 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/5fb72f117b644c2681b18b93ec9fff02 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/5fb72f117b644c2681b18b93ec9fff02 2024-11-28T06:54:17,848 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/2d8f7389c7d8493e87f24bc6ad773dc7 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/2d8f7389c7d8493e87f24bc6ad773dc7 2024-11-28T06:54:17,849 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/5f453a0e67ca4fc08c34db296ba72a97 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/5f453a0e67ca4fc08c34db296ba72a97 2024-11-28T06:54:17,851 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/952ec844c1214ce6be5d7f0af5060bb0 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/952ec844c1214ce6be5d7f0af5060bb0 2024-11-28T06:54:17,852 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/2dcc9874f8a142f4a8a91ecb5f77a169 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/2dcc9874f8a142f4a8a91ecb5f77a169 2024-11-28T06:54:17,853 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/4da13ca3cd584ecfacb1ea7666e3beb7 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/4da13ca3cd584ecfacb1ea7666e3beb7 2024-11-28T06:54:17,854 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/97cdfe8ad91e40e8b0015b19d6029c00 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/97cdfe8ad91e40e8b0015b19d6029c00 2024-11-28T06:54:17,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/a9ccb262218347de93b04e70f07f6a2f to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/a9ccb262218347de93b04e70f07f6a2f 2024-11-28T06:54:17,856 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/3445a1f8c1d948838ac0f2e743201542 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/3445a1f8c1d948838ac0f2e743201542 2024-11-28T06:54:17,857 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/a488ef25bf78450098e75eace8093d36 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/a488ef25bf78450098e75eace8093d36 2024-11-28T06:54:17,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/36762199ced842da9c739e66bfdd2dc9 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/36762199ced842da9c739e66bfdd2dc9 2024-11-28T06:54:17,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/95a65eaee9014cabb1121f33530f522d to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/95a65eaee9014cabb1121f33530f522d 2024-11-28T06:54:17,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/a21dece0323946528d5a0b9cd672a1ce to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/a21dece0323946528d5a0b9cd672a1ce 2024-11-28T06:54:17,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/9d3c459e37d344a481bb5fc10e583414 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/9d3c459e37d344a481bb5fc10e583414 2024-11-28T06:54:17,862 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/366541860f1c4026863a3f5032e56e72 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/366541860f1c4026863a3f5032e56e72 2024-11-28T06:54:17,863 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/6dd1cacc7c45461790141f41e9c8c48f to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/6dd1cacc7c45461790141f41e9c8c48f 2024-11-28T06:54:17,864 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/e5a48f79fe0040548e0babd968615db0 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/e5a48f79fe0040548e0babd968615db0 2024-11-28T06:54:17,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/99de9edc2c644e52a35fb298627b045e, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/9fac955e110c460ab5ae3168d460fe89, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/eff14829789e4911848f48dd092bd51f, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/f58dd164cd1e46278e7f234268740843, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/1396152e7dd941d9bda378b13b1c12ca, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/d2ae48590ba34986b7874734538f18a4, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/c91ab63f9f274667bf242ee4864b0ac9, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/c9cf2ddd2728415e9555f60b1cc68c0a, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/9d5e7da751bd4aeb9935e197f3d68a3b, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/b9d122b9ef0b4f6687361e34760d1d91, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/7fe75095178b4ed5b25fd13b367e60b6, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/2b71029d0173461595bcdebc6fabfc41, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/f90f2c59fa4c4f549a84677e09cdc585, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/4a33c7340fd84fb49a4b39f2f5c3cae4, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/63ef5eb3097f43b09cddb0cf36952969, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/30bd06de75f14d5ca8f6947d833f5042, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/3360b9cc25b34267a770ef07a6f5df07, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/d04e4c06476041d5a4ed53a4e5467be3, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/df5a104f58ad4f469e9522e7534c138f, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/10b6957509f4415395e42d3f83b256e1, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/af78906486b84ab4aad8346befaedbe1, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/d2c057985dd545ffa797712af3fe546c, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/bcdabece3c6440afb7ffaea818e5707e, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/5c738685dd644f2187993f39c4990e6f, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/84030f622ab9431eb8f56eb3a153de12, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/4ffaf290ff3c476fac72267c6382da0c, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/2177f68ad3644f9abffab236914e8a30, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/6d6719bcd7f9414f9c5d155b09c0a332, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/fce0dbe5289d4bc49f7893241dce974d, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/72d05257f24e42c7bb22f64e7ea8c97a, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/bf0806a51017421fafedc89966e54e19, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/97040d32c6054fbaab43606128f8a882] to archive 2024-11-28T06:54:17,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T06:54:17,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/99de9edc2c644e52a35fb298627b045e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/99de9edc2c644e52a35fb298627b045e 2024-11-28T06:54:17,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/9fac955e110c460ab5ae3168d460fe89 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/9fac955e110c460ab5ae3168d460fe89 2024-11-28T06:54:17,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/eff14829789e4911848f48dd092bd51f to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/eff14829789e4911848f48dd092bd51f 2024-11-28T06:54:17,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/f58dd164cd1e46278e7f234268740843 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/f58dd164cd1e46278e7f234268740843 2024-11-28T06:54:17,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/1396152e7dd941d9bda378b13b1c12ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/1396152e7dd941d9bda378b13b1c12ca 2024-11-28T06:54:17,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/d2ae48590ba34986b7874734538f18a4 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/d2ae48590ba34986b7874734538f18a4 2024-11-28T06:54:17,894 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/c91ab63f9f274667bf242ee4864b0ac9 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/c91ab63f9f274667bf242ee4864b0ac9 2024-11-28T06:54:17,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/c9cf2ddd2728415e9555f60b1cc68c0a to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/c9cf2ddd2728415e9555f60b1cc68c0a 2024-11-28T06:54:17,896 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/9d5e7da751bd4aeb9935e197f3d68a3b to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/9d5e7da751bd4aeb9935e197f3d68a3b 2024-11-28T06:54:17,897 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/b9d122b9ef0b4f6687361e34760d1d91 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/b9d122b9ef0b4f6687361e34760d1d91 2024-11-28T06:54:17,899 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/7fe75095178b4ed5b25fd13b367e60b6 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/7fe75095178b4ed5b25fd13b367e60b6 2024-11-28T06:54:17,900 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/2b71029d0173461595bcdebc6fabfc41 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/2b71029d0173461595bcdebc6fabfc41 2024-11-28T06:54:17,901 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/f90f2c59fa4c4f549a84677e09cdc585 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/f90f2c59fa4c4f549a84677e09cdc585 2024-11-28T06:54:17,902 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/4a33c7340fd84fb49a4b39f2f5c3cae4 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/4a33c7340fd84fb49a4b39f2f5c3cae4 2024-11-28T06:54:17,904 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/63ef5eb3097f43b09cddb0cf36952969 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/63ef5eb3097f43b09cddb0cf36952969 2024-11-28T06:54:17,905 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/30bd06de75f14d5ca8f6947d833f5042 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/30bd06de75f14d5ca8f6947d833f5042 2024-11-28T06:54:17,906 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/3360b9cc25b34267a770ef07a6f5df07 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/3360b9cc25b34267a770ef07a6f5df07 2024-11-28T06:54:17,907 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/d04e4c06476041d5a4ed53a4e5467be3 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/d04e4c06476041d5a4ed53a4e5467be3 2024-11-28T06:54:17,908 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/df5a104f58ad4f469e9522e7534c138f to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/df5a104f58ad4f469e9522e7534c138f 2024-11-28T06:54:17,909 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/10b6957509f4415395e42d3f83b256e1 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/10b6957509f4415395e42d3f83b256e1 2024-11-28T06:54:17,910 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/af78906486b84ab4aad8346befaedbe1 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/af78906486b84ab4aad8346befaedbe1 2024-11-28T06:54:17,911 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/d2c057985dd545ffa797712af3fe546c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/d2c057985dd545ffa797712af3fe546c 2024-11-28T06:54:17,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/bcdabece3c6440afb7ffaea818e5707e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/bcdabece3c6440afb7ffaea818e5707e 2024-11-28T06:54:17,913 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/5c738685dd644f2187993f39c4990e6f to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/5c738685dd644f2187993f39c4990e6f 2024-11-28T06:54:17,914 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/84030f622ab9431eb8f56eb3a153de12 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/84030f622ab9431eb8f56eb3a153de12 2024-11-28T06:54:17,916 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/4ffaf290ff3c476fac72267c6382da0c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/4ffaf290ff3c476fac72267c6382da0c 2024-11-28T06:54:17,917 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/2177f68ad3644f9abffab236914e8a30 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/2177f68ad3644f9abffab236914e8a30 2024-11-28T06:54:17,918 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/6d6719bcd7f9414f9c5d155b09c0a332 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/6d6719bcd7f9414f9c5d155b09c0a332 2024-11-28T06:54:17,919 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/fce0dbe5289d4bc49f7893241dce974d to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/fce0dbe5289d4bc49f7893241dce974d 2024-11-28T06:54:17,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/72d05257f24e42c7bb22f64e7ea8c97a to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/72d05257f24e42c7bb22f64e7ea8c97a 2024-11-28T06:54:17,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/bf0806a51017421fafedc89966e54e19 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/bf0806a51017421fafedc89966e54e19 2024-11-28T06:54:17,921 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/97040d32c6054fbaab43606128f8a882 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/97040d32c6054fbaab43606128f8a882 2024-11-28T06:54:17,923 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/6fc5d1b2bbed49fb85c747f961200581, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/7dc57186d65740a885469b06a0fa4d1e, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/8f815f8f4d2d45b7bbd88e4442ecaa04, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/8950d83064cd414c8748fab27cca64fd, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/5751bb49721c41288cd46ed0d62623dc, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/3fd11127f7614d42a5113894fdb0d78d, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/87bbb20d080c48498a0e5dea3eb7bdf6, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/c5be36c79319478db2d897a12d91904f, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/216e3d654da84f169259099463922641, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/4b608daed9b24b46897dafb85f89087a, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/81aaead0fcd345cea9debd442ddaaa19, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/125c741f1a7849ac86e68ade93170a66, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/2a02f310a51145479068ab9daf0278a7, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/6f3cff27e83b4604982f474f1e6b8851, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/fce399ca4f334463aa825024a75e3c58, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/cf3f7722dba946ffa5516cbff77db054, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/c3303330af7b439b95dde2ac665a8393, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/4fbc6f7e0f2d46209cf48ffaa9d1c52f, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/42df70b67fa14c0ab5e72a6e2fba70a0, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/301e23cfb9014642a998017b57fd1101, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/5dff3b621a864d57af9a1383acbd1b90, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/eeed1ec3f89243b3a1b5d6b46a54fee4, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/4540486ce28c4cca869cee5e2083afd4, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/3b68886cc2d84aca8c0a9de918f02ed7, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/c9a9fba8fc8440a18988b4e708b1ee53, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/dd099dabd4ec4d329bc0e9e44266bf85, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/38ac7a2a88864e4eaa1907599c63db44, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/f3fcd0b4262d4dddbe6994e8eb656f13, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/f8eb624f370345318aabb8a8fc34fda5, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/e604bd7d7620476cb7a193946c482a13, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/ca8adaf6464844a4ad873907e87278a0] to archive 2024-11-28T06:54:17,924 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T06:54:17,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/6fc5d1b2bbed49fb85c747f961200581 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/6fc5d1b2bbed49fb85c747f961200581 2024-11-28T06:54:17,926 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/7dc57186d65740a885469b06a0fa4d1e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/7dc57186d65740a885469b06a0fa4d1e 2024-11-28T06:54:17,927 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/8f815f8f4d2d45b7bbd88e4442ecaa04 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/8f815f8f4d2d45b7bbd88e4442ecaa04 2024-11-28T06:54:17,929 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/8950d83064cd414c8748fab27cca64fd to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/8950d83064cd414c8748fab27cca64fd 2024-11-28T06:54:17,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/5751bb49721c41288cd46ed0d62623dc to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/5751bb49721c41288cd46ed0d62623dc 2024-11-28T06:54:17,931 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/3fd11127f7614d42a5113894fdb0d78d to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/3fd11127f7614d42a5113894fdb0d78d 2024-11-28T06:54:17,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/87bbb20d080c48498a0e5dea3eb7bdf6 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/87bbb20d080c48498a0e5dea3eb7bdf6 2024-11-28T06:54:17,933 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/c5be36c79319478db2d897a12d91904f to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/c5be36c79319478db2d897a12d91904f 2024-11-28T06:54:17,934 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/216e3d654da84f169259099463922641 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/216e3d654da84f169259099463922641 2024-11-28T06:54:17,935 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/4b608daed9b24b46897dafb85f89087a to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/4b608daed9b24b46897dafb85f89087a 2024-11-28T06:54:17,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/81aaead0fcd345cea9debd442ddaaa19 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/81aaead0fcd345cea9debd442ddaaa19 2024-11-28T06:54:17,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/125c741f1a7849ac86e68ade93170a66 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/125c741f1a7849ac86e68ade93170a66 2024-11-28T06:54:17,938 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/2a02f310a51145479068ab9daf0278a7 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/2a02f310a51145479068ab9daf0278a7 2024-11-28T06:54:17,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/6f3cff27e83b4604982f474f1e6b8851 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/6f3cff27e83b4604982f474f1e6b8851 2024-11-28T06:54:17,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/fce399ca4f334463aa825024a75e3c58 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/fce399ca4f334463aa825024a75e3c58 2024-11-28T06:54:17,940 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/cf3f7722dba946ffa5516cbff77db054 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/cf3f7722dba946ffa5516cbff77db054 2024-11-28T06:54:17,941 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/c3303330af7b439b95dde2ac665a8393 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/c3303330af7b439b95dde2ac665a8393 2024-11-28T06:54:17,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/4fbc6f7e0f2d46209cf48ffaa9d1c52f to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/4fbc6f7e0f2d46209cf48ffaa9d1c52f 2024-11-28T06:54:17,943 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/42df70b67fa14c0ab5e72a6e2fba70a0 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/42df70b67fa14c0ab5e72a6e2fba70a0 2024-11-28T06:54:17,944 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/301e23cfb9014642a998017b57fd1101 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/301e23cfb9014642a998017b57fd1101 2024-11-28T06:54:17,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/5dff3b621a864d57af9a1383acbd1b90 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/5dff3b621a864d57af9a1383acbd1b90 2024-11-28T06:54:17,946 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/eeed1ec3f89243b3a1b5d6b46a54fee4 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/eeed1ec3f89243b3a1b5d6b46a54fee4 2024-11-28T06:54:17,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/4540486ce28c4cca869cee5e2083afd4 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/4540486ce28c4cca869cee5e2083afd4 2024-11-28T06:54:17,949 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/3b68886cc2d84aca8c0a9de918f02ed7 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/3b68886cc2d84aca8c0a9de918f02ed7 2024-11-28T06:54:17,950 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/c9a9fba8fc8440a18988b4e708b1ee53 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/c9a9fba8fc8440a18988b4e708b1ee53 2024-11-28T06:54:17,951 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/dd099dabd4ec4d329bc0e9e44266bf85 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/dd099dabd4ec4d329bc0e9e44266bf85 2024-11-28T06:54:17,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/38ac7a2a88864e4eaa1907599c63db44 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/38ac7a2a88864e4eaa1907599c63db44 2024-11-28T06:54:17,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/f3fcd0b4262d4dddbe6994e8eb656f13 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/f3fcd0b4262d4dddbe6994e8eb656f13 2024-11-28T06:54:17,954 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/f8eb624f370345318aabb8a8fc34fda5 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/f8eb624f370345318aabb8a8fc34fda5 2024-11-28T06:54:17,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/e604bd7d7620476cb7a193946c482a13 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/e604bd7d7620476cb7a193946c482a13 2024-11-28T06:54:17,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/ca8adaf6464844a4ad873907e87278a0 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/ca8adaf6464844a4ad873907e87278a0 2024-11-28T06:54:17,960 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/recovered.edits/445.seqid, newMaxSeqId=445, maxSeqId=1 2024-11-28T06:54:17,961 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. 2024-11-28T06:54:17,961 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1676): Region close journal for d6ca3b1ced864db1c50bb8f904a838f3: Waiting for close lock at 1732776856532Running coprocessor pre-close hooks at 1732776856532Disabling compacts and flushes for region at 1732776856532Disabling writes for close at 1732776856532Obtaining lock to block concurrent updates at 1732776856532Preparing flush snapshotting stores in d6ca3b1ced864db1c50bb8f904a838f3 at 1732776856532Finished memstore snapshotting TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732776856533 (+1 ms)Flushing stores of TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3. at 1732776856534 (+1 ms)Flushing d6ca3b1ced864db1c50bb8f904a838f3/A: creating writer at 1732776856534Flushing d6ca3b1ced864db1c50bb8f904a838f3/A: appending metadata at 1732776856539 (+5 ms)Flushing d6ca3b1ced864db1c50bb8f904a838f3/A: closing flushed file at 1732776856539Flushing d6ca3b1ced864db1c50bb8f904a838f3/B: creating writer at 1732776856957 (+418 ms)Flushing d6ca3b1ced864db1c50bb8f904a838f3/B: appending metadata at 1732776856961 (+4 ms)Flushing d6ca3b1ced864db1c50bb8f904a838f3/B: closing flushed file at 1732776856961Flushing d6ca3b1ced864db1c50bb8f904a838f3/C: creating writer at 1732776857374 (+413 ms)Flushing d6ca3b1ced864db1c50bb8f904a838f3/C: appending metadata at 1732776857378 (+4 ms)Flushing d6ca3b1ced864db1c50bb8f904a838f3/C: closing flushed file at 1732776857378Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@542ef78d: reopening flushed file at 1732776857793 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b25869e: reopening flushed file at 1732776857802 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3aec50e3: reopening flushed file at 1732776857808 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for d6ca3b1ced864db1c50bb8f904a838f3 in 1283ms, sequenceid=442, compaction requested=true at 1732776857815 (+7 ms)Writing region close event to WAL at 1732776857957 (+142 ms)Running coprocessor post-close hooks at 1732776857961 (+4 ms)Closed at 1732776857961 2024-11-28T06:54:17,964 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] handler.UnassignRegionHandler(157): Closed d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:54:17,964 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=d6ca3b1ced864db1c50bb8f904a838f3, regionState=CLOSED 2024-11-28T06:54:17,966 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE, hasLock=false; CloseRegionProcedure d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 because future has completed 2024-11-28T06:54:17,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-28T06:54:17,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; CloseRegionProcedure d6ca3b1ced864db1c50bb8f904a838f3, server=1cb9cf437a36,43341,1732776825568 in 1.5950 sec 2024-11-28T06:54:17,971 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-28T06:54:17,972 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d6ca3b1ced864db1c50bb8f904a838f3, UNASSIGN in 1.6020 sec 2024-11-28T06:54:17,974 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-28T06:54:17,974 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6090 sec 2024-11-28T06:54:17,976 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732776857976"}]},"ts":"1732776857976"} 2024-11-28T06:54:17,977 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-28T06:54:17,978 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-28T06:54:17,980 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6230 sec 2024-11-28T06:54:18,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-28T06:54:18,506 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:54:18,511 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-28T06:54:18,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T06:54:18,518 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=23, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T06:54:18,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-28T06:54:18,520 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=23, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T06:54:18,524 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:54:18,529 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A, FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B, FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C, FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/recovered.edits] 2024-11-28T06:54:18,533 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/8d310e22d2c74a80b688edd9e30fc2a3 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/8d310e22d2c74a80b688edd9e30fc2a3 2024-11-28T06:54:18,535 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/fc3193b37937486a8dadebb0df4e4338 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/fc3193b37937486a8dadebb0df4e4338 2024-11-28T06:54:18,536 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/fc5c8b149cef4b66bd78561abf5bf83d to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/A/fc5c8b149cef4b66bd78561abf5bf83d 2024-11-28T06:54:18,538 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/43c033f7b76e42c680701ee3029a939f to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/43c033f7b76e42c680701ee3029a939f 2024-11-28T06:54:18,540 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/810896d6d90149cab31fe06e54273914 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/810896d6d90149cab31fe06e54273914 2024-11-28T06:54:18,541 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/c486ba7fc94d40b7ab6a88e072b4fe5b to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/B/c486ba7fc94d40b7ab6a88e072b4fe5b 2024-11-28T06:54:18,543 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/0cbd0512d5d841d7a8cfee2ef1531339 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/0cbd0512d5d841d7a8cfee2ef1531339 2024-11-28T06:54:18,544 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/193ec61a1ce84d7b9e1aa5e974982e5c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/193ec61a1ce84d7b9e1aa5e974982e5c 2024-11-28T06:54:18,546 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/aa0cd0bceea1409fb900f0a0d1450266 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/C/aa0cd0bceea1409fb900f0a0d1450266 2024-11-28T06:54:18,549 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/recovered.edits/445.seqid to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3/recovered.edits/445.seqid 2024-11-28T06:54:18,549 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d6ca3b1ced864db1c50bb8f904a838f3 2024-11-28T06:54:18,549 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-28T06:54:18,555 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=23, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T06:54:18,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-28T06:54:18,564 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-28T06:54:18,594 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-28T06:54:18,596 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=23, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T06:54:18,596 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-28T06:54:18,596 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732776858596"}]},"ts":"9223372036854775807"} 2024-11-28T06:54:18,599 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-28T06:54:18,599 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d6ca3b1ced864db1c50bb8f904a838f3, NAME => 'TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3.', STARTKEY => '', ENDKEY => ''}] 2024-11-28T06:54:18,599 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-28T06:54:18,600 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732776858599"}]},"ts":"9223372036854775807"} 2024-11-28T06:54:18,602 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-28T06:54:18,603 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=23, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T06:54:18,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 91 msec 2024-11-28T06:54:18,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-28T06:54:18,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-28T06:54:18,627 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:54:18,644 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=245 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1249909516_22 at /127.0.0.1:53350 [Waiting for operation #104] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-682725700_22 at /127.0.0.1:59742 [Waiting for operation #274] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-682725700_22 at /127.0.0.1:52896 [Waiting for operation #265] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;1cb9cf437a36:43341-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=421 (was 389) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8086 (was 9155) 2024-11-28T06:54:18,653 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=245, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=421, ProcessCount=11, AvailableMemoryMB=8086 2024-11-28T06:54:18,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T06:54:18,655 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T06:54:18,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-28T06:54:18,657 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T06:54:18,658 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:18,658 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 24 2024-11-28T06:54:18,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-28T06:54:18,659 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T06:54:18,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741941_1117 (size=1061) 2024-11-28T06:54:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-28T06:54:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-28T06:54:19,072 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7 2024-11-28T06:54:19,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741942_1118 (size=53) 2024-11-28T06:54:19,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-28T06:54:19,515 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T06:54:19,516 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5f1556cdc2c5439575ba4a9f6cd46c4e, disabling compactions & flushes 2024-11-28T06:54:19,516 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:19,516 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:19,516 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. after waiting 0 ms 2024-11-28T06:54:19,516 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:19,516 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:19,516 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: Waiting for close lock at 1732776859515Disabling compacts and flushes for region at 1732776859516 (+1 ms)Disabling writes for close at 1732776859516Writing region close event to WAL at 1732776859516Closed at 1732776859516 2024-11-28T06:54:19,521 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T06:54:19,522 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732776859521"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732776859521"}]},"ts":"1732776859521"} 2024-11-28T06:54:19,525 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T06:54:19,526 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T06:54:19,527 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732776859526"}]},"ts":"1732776859526"} 2024-11-28T06:54:19,528 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-28T06:54:19,528 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f1556cdc2c5439575ba4a9f6cd46c4e, ASSIGN}] 2024-11-28T06:54:19,530 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f1556cdc2c5439575ba4a9f6cd46c4e, ASSIGN 2024-11-28T06:54:19,530 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f1556cdc2c5439575ba4a9f6cd46c4e, ASSIGN; state=OFFLINE, location=1cb9cf437a36,43341,1732776825568; forceNewPlan=false, retain=false 2024-11-28T06:54:19,682 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=5f1556cdc2c5439575ba4a9f6cd46c4e, regionState=OPENING, regionLocation=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:19,687 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f1556cdc2c5439575ba4a9f6cd46c4e, ASSIGN because future has completed 2024-11-28T06:54:19,689 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568}] 2024-11-28T06:54:19,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-28T06:54:19,848 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:19,848 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7752): Opening region: {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} 2024-11-28T06:54:19,849 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:19,849 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T06:54:19,849 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7794): checking encryption for 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:19,849 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7797): checking classloading for 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:19,851 INFO [StoreOpener-5f1556cdc2c5439575ba4a9f6cd46c4e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:19,853 INFO [StoreOpener-5f1556cdc2c5439575ba4a9f6cd46c4e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T06:54:19,854 INFO [StoreOpener-5f1556cdc2c5439575ba4a9f6cd46c4e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f1556cdc2c5439575ba4a9f6cd46c4e columnFamilyName A 2024-11-28T06:54:19,854 DEBUG [StoreOpener-5f1556cdc2c5439575ba4a9f6cd46c4e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:19,855 INFO [StoreOpener-5f1556cdc2c5439575ba4a9f6cd46c4e-1 {}] regionserver.HStore(327): Store=5f1556cdc2c5439575ba4a9f6cd46c4e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:54:19,855 INFO [StoreOpener-5f1556cdc2c5439575ba4a9f6cd46c4e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:19,856 INFO [StoreOpener-5f1556cdc2c5439575ba4a9f6cd46c4e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T06:54:19,856 INFO [StoreOpener-5f1556cdc2c5439575ba4a9f6cd46c4e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f1556cdc2c5439575ba4a9f6cd46c4e columnFamilyName B 2024-11-28T06:54:19,856 DEBUG [StoreOpener-5f1556cdc2c5439575ba4a9f6cd46c4e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:19,857 INFO [StoreOpener-5f1556cdc2c5439575ba4a9f6cd46c4e-1 {}] regionserver.HStore(327): Store=5f1556cdc2c5439575ba4a9f6cd46c4e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:54:19,857 INFO [StoreOpener-5f1556cdc2c5439575ba4a9f6cd46c4e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:19,858 INFO [StoreOpener-5f1556cdc2c5439575ba4a9f6cd46c4e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T06:54:19,858 INFO [StoreOpener-5f1556cdc2c5439575ba4a9f6cd46c4e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f1556cdc2c5439575ba4a9f6cd46c4e columnFamilyName C 2024-11-28T06:54:19,858 DEBUG [StoreOpener-5f1556cdc2c5439575ba4a9f6cd46c4e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:19,858 INFO [StoreOpener-5f1556cdc2c5439575ba4a9f6cd46c4e-1 {}] regionserver.HStore(327): Store=5f1556cdc2c5439575ba4a9f6cd46c4e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:54:19,859 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:19,859 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1038): replaying wal for 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:19,859 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:19,860 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:19,861 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1048): stopping wal replay for 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:19,861 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1060): Cleaning up temporary data for 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:19,861 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T06:54:19,863 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1093): writing seq id for 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:19,865 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T06:54:19,865 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1114): Opened 5f1556cdc2c5439575ba4a9f6cd46c4e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67407492, jitterRate=0.004449903964996338}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T06:54:19,865 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:19,866 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1006): Region open journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: Running coprocessor pre-open hook at 1732776859849Writing region info on filesystem at 1732776859849Initializing all the Stores at 1732776859851 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732776859851Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732776859851Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732776859851Cleaning up temporary data from old regions at 1732776859861 (+10 ms)Running coprocessor post-open hooks at 1732776859865 (+4 ms)Region opened successfully at 1732776859866 (+1 ms) 2024-11-28T06:54:19,867 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., pid=26, masterSystemTime=1732776859844 2024-11-28T06:54:19,869 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:19,869 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:19,869 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=5f1556cdc2c5439575ba4a9f6cd46c4e, regionState=OPEN, openSeqNum=2, regionLocation=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:19,871 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 because future has completed 2024-11-28T06:54:19,874 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-28T06:54:19,874 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 in 183 msec 2024-11-28T06:54:19,876 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-11-28T06:54:19,876 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f1556cdc2c5439575ba4a9f6cd46c4e, ASSIGN in 346 msec 2024-11-28T06:54:19,877 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T06:54:19,878 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732776859877"}]},"ts":"1732776859877"} 2024-11-28T06:54:19,879 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-28T06:54:19,880 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T06:54:19,882 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2250 sec 2024-11-28T06:54:20,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-28T06:54:20,806 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:54:20,807 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2697ccde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:20,807 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:54:20,808 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:54:20,809 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:54:20,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:54:20,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:54:20,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31989b2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:20,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:54:20,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:54:20,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:20,812 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52158, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:54:20,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c5b1922, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:20,813 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:54:20,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:54:20,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:54:20,817 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41346, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:54:20,820 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T06:54:20,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 1cb9cf437a36,37503,1732776824789 2024-11-28T06:54:20,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3815d824 2024-11-28T06:54:20,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T06:54:20,823 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52172, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T06:54:20,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T06:54:20,829 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T06:54:20,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-28T06:54:20,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741943_1119 (size=1097) 2024-11-28T06:54:21,254 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-28T06:54:21,254 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-28T06:54:21,258 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T06:54:21,264 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f1556cdc2c5439575ba4a9f6cd46c4e, REOPEN/MOVE}] 2024-11-28T06:54:21,265 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f1556cdc2c5439575ba4a9f6cd46c4e, REOPEN/MOVE 2024-11-28T06:54:21,266 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=5f1556cdc2c5439575ba4a9f6cd46c4e, regionState=CLOSING, regionLocation=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:21,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f1556cdc2c5439575ba4a9f6cd46c4e, REOPEN/MOVE because future has completed 2024-11-28T06:54:21,268 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T06:54:21,268 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568}] 2024-11-28T06:54:21,421 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:21,421 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T06:54:21,421 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 5f1556cdc2c5439575ba4a9f6cd46c4e, disabling compactions & flushes 2024-11-28T06:54:21,421 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:21,421 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:21,421 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. after waiting 0 ms 2024-11-28T06:54:21,422 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:21,429 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-28T06:54:21,430 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:21,430 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: Waiting for close lock at 1732776861421Running coprocessor pre-close hooks at 1732776861421Disabling compacts and flushes for region at 1732776861421Disabling writes for close at 1732776861422 (+1 ms)Writing region close event to WAL at 1732776861422Running coprocessor post-close hooks at 1732776861430 (+8 ms)Closed at 1732776861430 2024-11-28T06:54:21,431 WARN [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionServer(3299): Not adding moved region record: 5f1556cdc2c5439575ba4a9f6cd46c4e to self. 2024-11-28T06:54:21,433 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:21,434 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=5f1556cdc2c5439575ba4a9f6cd46c4e, regionState=CLOSED 2024-11-28T06:54:21,435 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 because future has completed 2024-11-28T06:54:21,438 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-28T06:54:21,438 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 in 168 msec 2024-11-28T06:54:21,439 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f1556cdc2c5439575ba4a9f6cd46c4e, REOPEN/MOVE; state=CLOSED, location=1cb9cf437a36,43341,1732776825568; forceNewPlan=false, retain=true 2024-11-28T06:54:21,590 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=5f1556cdc2c5439575ba4a9f6cd46c4e, regionState=OPENING, regionLocation=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:21,592 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f1556cdc2c5439575ba4a9f6cd46c4e, REOPEN/MOVE because future has completed 2024-11-28T06:54:21,592 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568}] 2024-11-28T06:54:21,753 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:21,753 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} 2024-11-28T06:54:21,754 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:21,754 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T06:54:21,754 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:21,754 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:21,758 INFO [StoreOpener-5f1556cdc2c5439575ba4a9f6cd46c4e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:21,759 INFO [StoreOpener-5f1556cdc2c5439575ba4a9f6cd46c4e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T06:54:21,766 INFO [StoreOpener-5f1556cdc2c5439575ba4a9f6cd46c4e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f1556cdc2c5439575ba4a9f6cd46c4e columnFamilyName A 2024-11-28T06:54:21,769 DEBUG [StoreOpener-5f1556cdc2c5439575ba4a9f6cd46c4e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:21,769 INFO [StoreOpener-5f1556cdc2c5439575ba4a9f6cd46c4e-1 {}] regionserver.HStore(327): Store=5f1556cdc2c5439575ba4a9f6cd46c4e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:54:21,770 INFO [StoreOpener-5f1556cdc2c5439575ba4a9f6cd46c4e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:21,770 INFO [StoreOpener-5f1556cdc2c5439575ba4a9f6cd46c4e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T06:54:21,771 INFO [StoreOpener-5f1556cdc2c5439575ba4a9f6cd46c4e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f1556cdc2c5439575ba4a9f6cd46c4e columnFamilyName B 2024-11-28T06:54:21,771 DEBUG [StoreOpener-5f1556cdc2c5439575ba4a9f6cd46c4e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:21,771 INFO [StoreOpener-5f1556cdc2c5439575ba4a9f6cd46c4e-1 {}] regionserver.HStore(327): Store=5f1556cdc2c5439575ba4a9f6cd46c4e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:54:21,771 INFO [StoreOpener-5f1556cdc2c5439575ba4a9f6cd46c4e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:21,772 INFO [StoreOpener-5f1556cdc2c5439575ba4a9f6cd46c4e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T06:54:21,772 INFO [StoreOpener-5f1556cdc2c5439575ba4a9f6cd46c4e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f1556cdc2c5439575ba4a9f6cd46c4e columnFamilyName C 2024-11-28T06:54:21,772 DEBUG [StoreOpener-5f1556cdc2c5439575ba4a9f6cd46c4e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:21,772 INFO [StoreOpener-5f1556cdc2c5439575ba4a9f6cd46c4e-1 {}] regionserver.HStore(327): Store=5f1556cdc2c5439575ba4a9f6cd46c4e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:54:21,772 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:21,773 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:21,773 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:21,774 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:21,775 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:21,775 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:21,776 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T06:54:21,777 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:21,778 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 5f1556cdc2c5439575ba4a9f6cd46c4e; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66377084, jitterRate=-0.010904371738433838}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T06:54:21,778 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:21,779 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: Running coprocessor pre-open hook at 1732776861754Writing region info on filesystem at 1732776861754Initializing all the Stores at 1732776861756 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732776861756Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732776861756Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732776861756Cleaning up temporary data from old regions at 1732776861775 (+19 ms)Running coprocessor post-open hooks at 1732776861778 (+3 ms)Region opened successfully at 1732776861779 (+1 ms) 2024-11-28T06:54:21,780 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., pid=31, masterSystemTime=1732776861744 2024-11-28T06:54:21,782 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:21,782 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:21,783 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=5f1556cdc2c5439575ba4a9f6cd46c4e, regionState=OPEN, openSeqNum=5, regionLocation=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:21,785 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 because future has completed 2024-11-28T06:54:21,788 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=29 2024-11-28T06:54:21,788 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 in 194 msec 2024-11-28T06:54:21,789 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-28T06:54:21,790 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f1556cdc2c5439575ba4a9f6cd46c4e, REOPEN/MOVE in 524 msec 2024-11-28T06:54:21,793 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-28T06:54:21,793 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 534 msec 2024-11-28T06:54:21,797 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 963 msec 2024-11-28T06:54:21,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-28T06:54:21,799 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:54:21,800 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:21,800 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:54:21,800 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:54:21,801 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:54:21,801 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:54:21,801 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:54:21,802 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40d7846f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:21,802 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:54:21,802 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:54:21,802 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:21,803 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52180, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:54:21,803 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73c17494, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:21,804 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35409f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:21,804 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:54:21,804 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:54:21,805 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:54:21,805 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:54:21,805 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:54:21,805 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46a2f761, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:21,805 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:54:21,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:54:21,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:21,806 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52202, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:54:21,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d226833, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:21,808 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7944f9cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:21,808 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:54:21,808 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:54:21,809 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:54:21,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:54:21,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:54:21,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fc11415, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:21,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:54:21,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:54:21,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:21,810 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52224, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:54:21,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f6b8f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:21,812 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38720fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:21,812 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:54:21,812 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:54:21,812 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:54:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:54:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:54:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dd6e88a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:54:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:54:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:21,814 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52238, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:54:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ee451e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:21,815 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c77b203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:21,815 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:54:21,815 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:54:21,816 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:54:21,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:54:21,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:54:21,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12896cbb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:21,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:54:21,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:54:21,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:21,817 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:54:21,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6837d6d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:21,819 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3f6eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:21,819 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:54:21,819 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:54:21,820 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:54:21,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:54:21,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:54:21,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45a9ae94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:21,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:54:21,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:54:21,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:21,821 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52266, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:54:21,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d520a83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:21,823 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40353459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:21,823 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:54:21,823 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:54:21,824 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:54:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:54:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:54:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c046bf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:54:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:54:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:21,825 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52276, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:54:21,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d8d12f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:21,826 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@160383af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:21,826 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:54:21,827 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:54:21,827 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:54:21,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:54:21,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:54:21,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d487d27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:21,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:54:21,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:54:21,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:21,828 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52298, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:54:21,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59dc355c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:21,830 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@123a694c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:21,830 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:54:21,830 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:54:21,830 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:54:21,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:54:21,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:54:21,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@301d19fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:21,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:54:21,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:54:21,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:21,832 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52322, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:54:21,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dbf7061, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:21,834 DEBUG [Thread-611 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:54:21,834 DEBUG [Thread-615 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:54:21,834 DEBUG [Thread-624 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:54:21,834 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:54:21,835 DEBUG [Thread-607 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:54:21,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:54:21,835 DEBUG [Thread-620 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:54:21,835 DEBUG [Thread-613 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:54:21,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:54:21,835 DEBUG [Thread-618 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:54:21,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:54:21,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:54:21,837 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:54:21,837 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41362, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:54:21,837 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:54:21,838 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41364, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:54:21,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:54:21,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees 2024-11-28T06:54:21,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:54:21,840 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41374, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:54:21,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:54:21,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:54:21,841 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=32, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:54:21,841 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:54:21,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:54:21,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:54:21,842 DEBUG [Thread-622 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:54:21,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:54:21,842 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41388, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:54:21,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5] 2024-11-28T06:54:21,842 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41394, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:54:21,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5] 2024-11-28T06:54:21,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:54:21,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:54:21,844 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5] 2024-11-28T06:54:21,844 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:54:21,844 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41410, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:54:21,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-28T06:54:21,844 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41426, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:54:21,845 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=32, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:54:21,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:54:21,845 DEBUG [Thread-609 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:54:21,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5] 2024-11-28T06:54:21,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5] 2024-11-28T06:54:21,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:54:21,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5] 2024-11-28T06:54:21,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:54:21,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5] 2024-11-28T06:54:21,852 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41430, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:54:21,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5] 2024-11-28T06:54:21,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5] 2024-11-28T06:54:21,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:21,861 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f1556cdc2c5439575ba4a9f6cd46c4e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T06:54:21,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=A 2024-11-28T06:54:21,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:21,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=B 2024-11-28T06:54:21,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:21,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=C 2024-11-28T06:54:21,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:21,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16434, totalHeapSize=37576, min timestamp=1732776861879, max timestamp=1732776861915Num uniques 859; 2024-11-28T06:54:21,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=16434, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-28T06:54:21,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411288538eb3c7fd54833af3b4186ca2e8852_5f1556cdc2c5439575ba4a9f6cd46c4e is 50, key is test_row_0/A:col10/1732776861860/Put/seqid=0 2024-11-28T06:54:21,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732776861879, max timestamp=1732776861915Num uniques 859; 2024-11-28T06:54:21,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-28T06:54:21,935 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732776861907, max timestamp=1732776861915Num uniques 859; 2024-11-28T06:54:21,935 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-28T06:54:21,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741944_1120 (size=14594) 2024-11-28T06:54:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-28T06:54:21,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-28T06:54:21,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:21,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:21,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:21,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:21,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:22,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:22,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:22,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776922006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41388 deadline: 1732776922006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:22,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:22,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:22,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776922010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:22,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:22,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:22,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:22,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:22,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776922017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:22,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41430 deadline: 1732776922018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:22,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:22,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:22,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:22,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776922126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:22,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:22,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41388 deadline: 1732776922126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:22,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:22,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776922127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:22,133 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,133 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,133 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:22,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:22,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:22,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41430 deadline: 1732776922145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:22,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:22,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776922146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:22,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:22,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:22,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-28T06:54:22,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:22,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:22,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:22,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:22,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:22,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-28T06:54:22,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-28T06:54:22,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:22,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:22,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:22,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:22,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:22,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:22,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776922337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:22,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:22,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41388 deadline: 1732776922337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:22,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:22,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:22,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776922337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:22,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:22,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:22,346 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:22,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41430 deadline: 1732776922359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:22,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776922359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:22,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:22,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:22,374 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411288538eb3c7fd54833af3b4186ca2e8852_5f1556cdc2c5439575ba4a9f6cd46c4e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288538eb3c7fd54833af3b4186ca2e8852_5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:22,377 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/dd04892fa1bb46f78edc42b7eea22fc5, store: [table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:22,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/dd04892fa1bb46f78edc42b7eea22fc5 is 175, key is test_row_0/A:col10/1732776861860/Put/seqid=0 2024-11-28T06:54:22,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741945_1121 (size=39549) 2024-11-28T06:54:22,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-28T06:54:22,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:22,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:22,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:22,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:22,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:22,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-28T06:54:22,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-28T06:54:22,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:22,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:22,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:22,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:22,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:22,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41388 deadline: 1732776922646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:22,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:22,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776922646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:22,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:22,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:22,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776922646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:22,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:22,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:22,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:22,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41430 deadline: 1732776922666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:22,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:22,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:22,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776922666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:22,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:22,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:22,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-28T06:54:22,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:22,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:22,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:22,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:22,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:22,818 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/dd04892fa1bb46f78edc42b7eea22fc5 2024-11-28T06:54:22,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/b75accbaf9ed4e8c9c5ae82d606e0536 is 50, key is test_row_0/B:col10/1732776861860/Put/seqid=0 2024-11-28T06:54:22,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741946_1122 (size=12001) 2024-11-28T06:54:22,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/b75accbaf9ed4e8c9c5ae82d606e0536 2024-11-28T06:54:22,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/d73c3d060ff54a0eb1277fe80feb4ed9 is 50, key is test_row_0/C:col10/1732776861860/Put/seqid=0 2024-11-28T06:54:22,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741947_1123 (size=12001) 2024-11-28T06:54:22,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-28T06:54:22,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:22,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:22,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:22,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:22,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-28T06:54:23,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-28T06:54:23,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:23,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:23,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:23,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:23,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:23,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41388 deadline: 1732776923155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:23,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:23,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:23,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776923155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:23,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:23,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776923155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:23,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41430 deadline: 1732776923176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:23,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:23,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:23,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:23,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776923176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:23,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:23,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:23,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:23,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-28T06:54:23,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:23,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:23,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:23,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:23,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:23,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/d73c3d060ff54a0eb1277fe80feb4ed9 2024-11-28T06:54:23,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/dd04892fa1bb46f78edc42b7eea22fc5 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/dd04892fa1bb46f78edc42b7eea22fc5 2024-11-28T06:54:23,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/dd04892fa1bb46f78edc42b7eea22fc5, entries=200, sequenceid=16, filesize=38.6 K 2024-11-28T06:54:23,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/b75accbaf9ed4e8c9c5ae82d606e0536 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/b75accbaf9ed4e8c9c5ae82d606e0536 2024-11-28T06:54:23,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/b75accbaf9ed4e8c9c5ae82d606e0536, entries=150, sequenceid=16, filesize=11.7 K 2024-11-28T06:54:23,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/d73c3d060ff54a0eb1277fe80feb4ed9 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/d73c3d060ff54a0eb1277fe80feb4ed9 2024-11-28T06:54:23,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/d73c3d060ff54a0eb1277fe80feb4ed9, entries=150, sequenceid=16, filesize=11.7 K 2024-11-28T06:54:23,347 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=131.55 KB/134705 for 5f1556cdc2c5439575ba4a9f6cd46c4e in 1486ms, sequenceid=16, compaction requested=false 2024-11-28T06:54:23,347 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-28T06:54:23,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: 2024-11-28T06:54:23,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-28T06:54:23,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:23,382 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2902): Flushing 5f1556cdc2c5439575ba4a9f6cd46c4e 3/3 column families, dataSize=131.55 KB heapSize=364.93 KB 2024-11-28T06:54:23,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=A 2024-11-28T06:54:23,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:54:23,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=B 2024-11-28T06:54:23,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:54:23,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=C 2024-11-28T06:54:23,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:54:23,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411280ab2ac28729740059e64a52203b28f79_5f1556cdc2c5439575ba4a9f6cd46c4e is 50, key is test_row_0/A:col10/1732776862014/Put/seqid=0 2024-11-28T06:54:23,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741948_1124 (size=12154) 2024-11-28T06:54:23,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,423 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411280ab2ac28729740059e64a52203b28f79_5f1556cdc2c5439575ba4a9f6cd46c4e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411280ab2ac28729740059e64a52203b28f79_5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:23,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/a6a388b5cffb4be380b242043353f613, store: [table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:23,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/a6a388b5cffb4be380b242043353f613 is 175, key is test_row_0/A:col10/1732776862014/Put/seqid=0 2024-11-28T06:54:23,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741949_1125 (size=30955) 2024-11-28T06:54:23,431 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/a6a388b5cffb4be380b242043353f613 2024-11-28T06:54:23,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/0224219bb6944d8080cc383a28ad2391 is 50, key is test_row_0/B:col10/1732776862014/Put/seqid=0 2024-11-28T06:54:23,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741950_1126 (size=12001) 2024-11-28T06:54:23,453 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/0224219bb6944d8080cc383a28ad2391 2024-11-28T06:54:23,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/dc288fbf325d44c5ab100bd149120dc4 is 50, key is test_row_0/C:col10/1732776862014/Put/seqid=0 2024-11-28T06:54:23,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741951_1127 (size=12001) 2024-11-28T06:54:23,490 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T06:54:23,868 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=37.62 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/dc288fbf325d44c5ab100bd149120dc4 2024-11-28T06:54:23,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/a6a388b5cffb4be380b242043353f613 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/a6a388b5cffb4be380b242043353f613 2024-11-28T06:54:23,896 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/a6a388b5cffb4be380b242043353f613, entries=150, sequenceid=50, filesize=30.2 K 2024-11-28T06:54:23,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/0224219bb6944d8080cc383a28ad2391 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/0224219bb6944d8080cc383a28ad2391 2024-11-28T06:54:23,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,906 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/0224219bb6944d8080cc383a28ad2391, entries=150, sequenceid=50, filesize=11.7 K 2024-11-28T06:54:23,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/dc288fbf325d44c5ab100bd149120dc4 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/dc288fbf325d44c5ab100bd149120dc4 2024-11-28T06:54:23,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,914 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/dc288fbf325d44c5ab100bd149120dc4, entries=150, sequenceid=50, filesize=11.7 K 2024-11-28T06:54:23,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,916 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(3140): Finished flush of dataSize ~131.55 KB/134705, heapSize ~364.88 KB/373640, currentSize=0 B/0 for 5f1556cdc2c5439575ba4a9f6cd46c4e in 533ms, sequenceid=50, compaction requested=false 2024-11-28T06:54:23,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2603): Flush status journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: 2024-11-28T06:54:23,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:23,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=33 2024-11-28T06:54:23,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=33 2024-11-28T06:54:23,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,920 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-28T06:54:23,920 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0730 sec 2024-11-28T06:54:23,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,923 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees in 2.0870 sec 2024-11-28T06:54:23,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-28T06:54:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,996 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:54:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,997 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:54:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees 2024-11-28T06:54:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-28T06:54:23,999 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:54:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,003 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:54:24,003 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:54:24,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-28T06:54:24,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-28T06:54:24,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:24,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: 2024-11-28T06:54:24,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:24,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-11-28T06:54:24,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-11-28T06:54:24,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,168 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-28T06:54:24,168 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 162 msec 2024-11-28T06:54:24,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,171 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees in 172 msec 2024-11-28T06:54:24,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:24,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f1556cdc2c5439575ba4a9f6cd46c4e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T06:54:24,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=A 2024-11-28T06:54:24,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:24,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=B 2024-11-28T06:54:24,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:24,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=C 2024-11-28T06:54:24,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:24,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112822101c82a3bb448b85cb648c5fb11a72_5f1556cdc2c5439575ba4a9f6cd46c4e is 50, key is test_row_0/A:col10/1732776864186/Put/seqid=0 2024-11-28T06:54:24,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741952_1128 (size=21918) 2024-11-28T06:54:24,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1732776864224, max timestamp=1732776864300Num uniques 859; 2024-11-28T06:54:24,305 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1732776864224, max timestamp=1732776864300Num uniques 859; 2024-11-28T06:54:24,305 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-28T06:54:24,305 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-28T06:54:24,305 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1732776864224, max timestamp=1732776864300Num uniques 859; 2024-11-28T06:54:24,305 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-28T06:54:24,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:24,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776924309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:24,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:24,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-28T06:54:24,316 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:54:24,317 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:54:24,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-28T06:54:24,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:24,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776924312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:24,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-28T06:54:24,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:24,320 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:54:24,321 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:54:24,321 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:54:24,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:24,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41430 deadline: 1732776924313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:24,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:24,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776924314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:24,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:24,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41388 deadline: 1732776924314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:24,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:24,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:24,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:24,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-28T06:54:24,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:24,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776924426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:24,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:24,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:24,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776924426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:24,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:24,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:24,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776924435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:24,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:24,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:24,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41388 deadline: 1732776924436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:24,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:24,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41430 deadline: 1732776924436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:24,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:24,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:24,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-28T06:54:24,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:24,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:24,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:24,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:24,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:24,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:24,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-28T06:54:24,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:24,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:24,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:24,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:24,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:24,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:24,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-28T06:54:24,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:24,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:24,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776924636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:24,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776924636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:24,643 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,643 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,643 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:24,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:24,659 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:24,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:24,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776924656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:24,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:24,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:24,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:24,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41430 deadline: 1732776924656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:24,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41388 deadline: 1732776924656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:24,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:24,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:24,665 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112822101c82a3bb448b85cb648c5fb11a72_5f1556cdc2c5439575ba4a9f6cd46c4e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112822101c82a3bb448b85cb648c5fb11a72_5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:24,666 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/0cb5aad427f041519f56376cf53ed968, store: [table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:24,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/0cb5aad427f041519f56376cf53ed968 is 175, key is test_row_0/A:col10/1732776864186/Put/seqid=0 2024-11-28T06:54:24,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741953_1129 (size=65323) 2024-11-28T06:54:24,681 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/0cb5aad427f041519f56376cf53ed968 2024-11-28T06:54:24,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/5d3e83500b684d8fad7fd8d70c9816ca is 50, key is test_row_0/B:col10/1732776864186/Put/seqid=0 2024-11-28T06:54:24,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741954_1130 (size=12001) 2024-11-28T06:54:24,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-28T06:54:24,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:24,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:24,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:24,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:24,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:24,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-28T06:54:24,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:24,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:24,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:24,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:24,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:24,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:24,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-28T06:54:24,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:24,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776924946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:24,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:24,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:24,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776924956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:24,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:24,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:24,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776924966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:24,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:24,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:24,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41430 deadline: 1732776924976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:24,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:24,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41388 deadline: 1732776924976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:24,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:24,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:24,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:25,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-28T06:54:25,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:25,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:25,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:25,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:25,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:25,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:25,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/5d3e83500b684d8fad7fd8d70c9816ca 2024-11-28T06:54:25,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/9573ee658a994b79b8934b19fdfa25c0 is 50, key is test_row_0/C:col10/1732776864186/Put/seqid=0 2024-11-28T06:54:25,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741955_1131 (size=12001) 2024-11-28T06:54:25,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/9573ee658a994b79b8934b19fdfa25c0 2024-11-28T06:54:25,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/0cb5aad427f041519f56376cf53ed968 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/0cb5aad427f041519f56376cf53ed968 2024-11-28T06:54:25,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/0cb5aad427f041519f56376cf53ed968, entries=350, sequenceid=61, filesize=63.8 K 2024-11-28T06:54:25,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/5d3e83500b684d8fad7fd8d70c9816ca as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/5d3e83500b684d8fad7fd8d70c9816ca 2024-11-28T06:54:25,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/5d3e83500b684d8fad7fd8d70c9816ca, entries=150, sequenceid=61, filesize=11.7 K 2024-11-28T06:54:25,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/9573ee658a994b79b8934b19fdfa25c0 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/9573ee658a994b79b8934b19fdfa25c0 2024-11-28T06:54:25,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/9573ee658a994b79b8934b19fdfa25c0, entries=150, sequenceid=61, filesize=11.7 K 2024-11-28T06:54:25,160 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=154.31 KB/158010 for 5f1556cdc2c5439575ba4a9f6cd46c4e in 937ms, sequenceid=61, compaction requested=true 2024-11-28T06:54:25,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: 2024-11-28T06:54:25,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1556cdc2c5439575ba4a9f6cd46c4e:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:54:25,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:25,161 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:54:25,161 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:54:25,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1556cdc2c5439575ba4a9f6cd46c4e:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:54:25,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:25,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1556cdc2c5439575ba4a9f6cd46c4e:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:54:25,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:54:25,163 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:54:25,163 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): 5f1556cdc2c5439575ba4a9f6cd46c4e/B is initiating minor compaction (all files) 2024-11-28T06:54:25,163 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1556cdc2c5439575ba4a9f6cd46c4e/B in TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:25,163 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/b75accbaf9ed4e8c9c5ae82d606e0536, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/0224219bb6944d8080cc383a28ad2391, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/5d3e83500b684d8fad7fd8d70c9816ca] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp, totalSize=35.2 K 2024-11-28T06:54:25,163 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 135827 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:54:25,163 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): 5f1556cdc2c5439575ba4a9f6cd46c4e/A is initiating minor compaction (all files) 2024-11-28T06:54:25,164 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1556cdc2c5439575ba4a9f6cd46c4e/A in TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:25,164 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/dd04892fa1bb46f78edc42b7eea22fc5, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/a6a388b5cffb4be380b242043353f613, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/0cb5aad427f041519f56376cf53ed968] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp, totalSize=132.6 K 2024-11-28T06:54:25,164 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:25,164 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. files: [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/dd04892fa1bb46f78edc42b7eea22fc5, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/a6a388b5cffb4be380b242043353f613, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/0cb5aad427f041519f56376cf53ed968] 2024-11-28T06:54:25,165 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting b75accbaf9ed4e8c9c5ae82d606e0536, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732776861848 2024-11-28T06:54:25,165 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd04892fa1bb46f78edc42b7eea22fc5, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732776861848 2024-11-28T06:54:25,166 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 0224219bb6944d8080cc383a28ad2391, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732776862003 2024-11-28T06:54:25,166 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6a388b5cffb4be380b242043353f613, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732776862003 2024-11-28T06:54:25,166 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d3e83500b684d8fad7fd8d70c9816ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732776864186 2024-11-28T06:54:25,167 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cb5aad427f041519f56376cf53ed968, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732776864186 2024-11-28T06:54:25,179 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:25,190 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1556cdc2c5439575ba4a9f6cd46c4e#B#compaction#114 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:54:25,190 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/8f36073e8c314e1481a7175a29c96855 is 50, key is test_row_0/B:col10/1732776864186/Put/seqid=0 2024-11-28T06:54:25,195 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411280362c51d697f4bf38ce0f19f246b7347_5f1556cdc2c5439575ba4a9f6cd46c4e store=[table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:25,198 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411280362c51d697f4bf38ce0f19f246b7347_5f1556cdc2c5439575ba4a9f6cd46c4e, store=[table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:25,199 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411280362c51d697f4bf38ce0f19f246b7347_5f1556cdc2c5439575ba4a9f6cd46c4e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:25,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741956_1132 (size=12104) 2024-11-28T06:54:25,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741957_1133 (size=4469) 2024-11-28T06:54:25,223 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1556cdc2c5439575ba4a9f6cd46c4e#A#compaction#113 average throughput is 0.56 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:54:25,224 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/d1d75e9b08024be497a6b0ad114d2362 is 175, key is test_row_0/A:col10/1732776864186/Put/seqid=0 2024-11-28T06:54:25,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741958_1134 (size=31058) 2024-11-28T06:54:25,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-28T06:54:25,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:25,246 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 5f1556cdc2c5439575ba4a9f6cd46c4e 3/3 column families, dataSize=154.31 KB heapSize=379.13 KB 2024-11-28T06:54:25,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=A 2024-11-28T06:54:25,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:54:25,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=B 2024-11-28T06:54:25,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:54:25,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=C 2024-11-28T06:54:25,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:54:25,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411288b2ea539b40e4204b585e4330f66b87d_5f1556cdc2c5439575ba4a9f6cd46c4e is 50, key is test_row_0/A:col10/1732776864300/Put/seqid=0 2024-11-28T06:54:25,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741959_1135 (size=12154) 2024-11-28T06:54:25,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:25,276 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411288b2ea539b40e4204b585e4330f66b87d_5f1556cdc2c5439575ba4a9f6cd46c4e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288b2ea539b40e4204b585e4330f66b87d_5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:25,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/1bb6e60b2f254e06927e2fee4eb7345f, store: [table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:25,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/1bb6e60b2f254e06927e2fee4eb7345f is 175, key is test_row_0/A:col10/1732776864300/Put/seqid=0 2024-11-28T06:54:25,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741960_1136 (size=30955) 2024-11-28T06:54:25,297 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=87, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/1bb6e60b2f254e06927e2fee4eb7345f 2024-11-28T06:54:25,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/4a3e83ff6f40448a9314cc9d05aba8f4 is 50, key is test_row_0/B:col10/1732776864300/Put/seqid=0 2024-11-28T06:54:25,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741961_1137 (size=12001) 2024-11-28T06:54:25,325 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/4a3e83ff6f40448a9314cc9d05aba8f4 2024-11-28T06:54:25,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/5a7de5b675d84610b0a00cb20cda66ee is 50, key is test_row_0/C:col10/1732776864300/Put/seqid=0 2024-11-28T06:54:25,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741962_1138 (size=12001) 2024-11-28T06:54:25,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-28T06:54:25,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:25,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:25,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:25,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776925483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:25,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:25,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776925484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:25,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:25,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:25,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:25,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:25,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776925485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:25,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:25,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:25,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:25,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41388 deadline: 1732776925485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:25,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:25,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:25,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:25,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:25,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:25,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:25,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:25,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:25,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41430 deadline: 1732776925496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:25,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:25,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:25,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:25,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:25,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776925596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:25,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:25,598 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:25,598 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:25,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776925596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:25,598 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:25,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:25,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776925596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:25,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:25,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:25,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:25,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:25,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:25,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:25,621 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/8f36073e8c314e1481a7175a29c96855 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/8f36073e8c314e1481a7175a29c96855 2024-11-28T06:54:25,626 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f1556cdc2c5439575ba4a9f6cd46c4e/B of 5f1556cdc2c5439575ba4a9f6cd46c4e into 8f36073e8c314e1481a7175a29c96855(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:25,626 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: 2024-11-28T06:54:25,626 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., storeName=5f1556cdc2c5439575ba4a9f6cd46c4e/B, priority=13, startTime=1732776865161; duration=0sec 2024-11-28T06:54:25,626 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:54:25,626 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1556cdc2c5439575ba4a9f6cd46c4e:B 2024-11-28T06:54:25,626 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:54:25,627 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:54:25,628 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): 5f1556cdc2c5439575ba4a9f6cd46c4e/C is initiating minor compaction (all files) 2024-11-28T06:54:25,628 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1556cdc2c5439575ba4a9f6cd46c4e/C in TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:25,628 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/d73c3d060ff54a0eb1277fe80feb4ed9, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/dc288fbf325d44c5ab100bd149120dc4, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/9573ee658a994b79b8934b19fdfa25c0] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp, totalSize=35.2 K 2024-11-28T06:54:25,628 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting d73c3d060ff54a0eb1277fe80feb4ed9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732776861848 2024-11-28T06:54:25,628 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting dc288fbf325d44c5ab100bd149120dc4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732776862003 2024-11-28T06:54:25,629 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 9573ee658a994b79b8934b19fdfa25c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732776864186 2024-11-28T06:54:25,635 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/d1d75e9b08024be497a6b0ad114d2362 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/d1d75e9b08024be497a6b0ad114d2362 2024-11-28T06:54:25,638 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1556cdc2c5439575ba4a9f6cd46c4e#C#compaction#118 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:54:25,638 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/7fea1ff5423640d08f15382829855ff1 is 50, key is test_row_0/C:col10/1732776864186/Put/seqid=0 2024-11-28T06:54:25,641 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f1556cdc2c5439575ba4a9f6cd46c4e/A of 5f1556cdc2c5439575ba4a9f6cd46c4e into d1d75e9b08024be497a6b0ad114d2362(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:25,641 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: 2024-11-28T06:54:25,641 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., storeName=5f1556cdc2c5439575ba4a9f6cd46c4e/A, priority=13, startTime=1732776865160; duration=0sec 2024-11-28T06:54:25,642 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:25,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741963_1139 (size=12104) 2024-11-28T06:54:25,642 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1556cdc2c5439575ba4a9f6cd46c4e:A 2024-11-28T06:54:25,761 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/5a7de5b675d84610b0a00cb20cda66ee 2024-11-28T06:54:25,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/1bb6e60b2f254e06927e2fee4eb7345f as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/1bb6e60b2f254e06927e2fee4eb7345f 2024-11-28T06:54:25,772 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/1bb6e60b2f254e06927e2fee4eb7345f, entries=150, sequenceid=87, filesize=30.2 K 2024-11-28T06:54:25,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/4a3e83ff6f40448a9314cc9d05aba8f4 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/4a3e83ff6f40448a9314cc9d05aba8f4 2024-11-28T06:54:25,777 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/4a3e83ff6f40448a9314cc9d05aba8f4, entries=150, sequenceid=87, filesize=11.7 K 2024-11-28T06:54:25,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/5a7de5b675d84610b0a00cb20cda66ee as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/5a7de5b675d84610b0a00cb20cda66ee 2024-11-28T06:54:25,781 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/5a7de5b675d84610b0a00cb20cda66ee, entries=150, sequenceid=87, filesize=11.7 K 2024-11-28T06:54:25,782 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~379.08 KB/388176, currentSize=53.67 KB/54960 for 5f1556cdc2c5439575ba4a9f6cd46c4e in 537ms, sequenceid=87, compaction requested=false 2024-11-28T06:54:25,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: 2024-11-28T06:54:25,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:25,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-28T06:54:25,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-28T06:54:25,786 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-28T06:54:25,786 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4630 sec 2024-11-28T06:54:25,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 1.4690 sec 2024-11-28T06:54:25,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:25,811 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f1556cdc2c5439575ba4a9f6cd46c4e 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T06:54:25,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=A 2024-11-28T06:54:25,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:25,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=B 2024-11-28T06:54:25,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:25,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=C 2024-11-28T06:54:25,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:25,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112862c2ba81a1fe412da6a0259d61c2ebc1_5f1556cdc2c5439575ba4a9f6cd46c4e is 50, key is test_row_0/A:col10/1732776865809/Put/seqid=0 2024-11-28T06:54:25,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741964_1140 (size=12154) 2024-11-28T06:54:25,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:25,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776925876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:25,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:25,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:25,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:25,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:25,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776925883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:25,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:25,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776925884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:25,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:25,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:25,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:25,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:25,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776925996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:25,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:25,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:25,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:25,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:25,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776925996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:25,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:25,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:25,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:25,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:25,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776925996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:25,999 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:25,999 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:25,999 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:26,051 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/7fea1ff5423640d08f15382829855ff1 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/7fea1ff5423640d08f15382829855ff1 2024-11-28T06:54:26,061 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f1556cdc2c5439575ba4a9f6cd46c4e/C of 5f1556cdc2c5439575ba4a9f6cd46c4e into 7fea1ff5423640d08f15382829855ff1(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:26,061 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: 2024-11-28T06:54:26,061 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., storeName=5f1556cdc2c5439575ba4a9f6cd46c4e/C, priority=13, startTime=1732776865161; duration=0sec 2024-11-28T06:54:26,061 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:26,061 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1556cdc2c5439575ba4a9f6cd46c4e:C 2024-11-28T06:54:26,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:26,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776926205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:26,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:26,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:26,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:26,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:26,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776926206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:26,207 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:26,207 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:26,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:26,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776926206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:26,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:26,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:26,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:26,222 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:26,227 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112862c2ba81a1fe412da6a0259d61c2ebc1_5f1556cdc2c5439575ba4a9f6cd46c4e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112862c2ba81a1fe412da6a0259d61c2ebc1_5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:26,229 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/03ca83dcf1a94b78b28f09e120b2098f, store: [table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:26,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/03ca83dcf1a94b78b28f09e120b2098f is 175, key is test_row_0/A:col10/1732776865809/Put/seqid=0 2024-11-28T06:54:26,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741965_1141 (size=30955) 2024-11-28T06:54:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-28T06:54:26,466 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:54:26,466 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:54:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-28T06:54:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-28T06:54:26,469 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:54:26,469 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:54:26,470 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:54:26,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41388 deadline: 1732776926506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:26,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41430 deadline: 1732776926506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:26,507 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_2' in 5f1556cdc2c5439575ba4a9f6cd46c4e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:54:26,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:26,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:26,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:26,508 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_0' in 5f1556cdc2c5439575ba4a9f6cd46c4e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:54:26,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:26,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:26,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:26,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776926515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:26,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:26,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:26,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776926515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:26,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:26,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:26,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:26,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:26,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776926516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:26,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:26,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:26,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:26,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-28T06:54:26,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-28T06:54:26,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:26,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:26,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:26,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:26,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:26,634 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=103, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/03ca83dcf1a94b78b28f09e120b2098f 2024-11-28T06:54:26,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/258f04dd2eb848ba8bf6d0accef72e87 is 50, key is test_row_0/B:col10/1732776865809/Put/seqid=0 2024-11-28T06:54:26,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741966_1142 (size=12001) 2024-11-28T06:54:26,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-28T06:54:26,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:26,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:26,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:26,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:26,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:26,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:26,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-28T06:54:26,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-28T06:54:26,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:26,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:26,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:26,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:26,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:27,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776927026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:27,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:27,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776927026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:27,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:27,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:27,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:27,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:27,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776927026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:27,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:27,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:27,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:27,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:27,050 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/258f04dd2eb848ba8bf6d0accef72e87 2024-11-28T06:54:27,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/e4c757d6a8e1485294dc12e68f47d530 is 50, key is test_row_0/C:col10/1732776865809/Put/seqid=0 2024-11-28T06:54:27,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741967_1143 (size=12001) 2024-11-28T06:54:27,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-28T06:54:27,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:27,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:27,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:27,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:27,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-28T06:54:27,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-28T06:54:27,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:27,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:27,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:27,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:27,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:27,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-28T06:54:27,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:27,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:27,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:27,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:27,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:27,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:27,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/e4c757d6a8e1485294dc12e68f47d530 2024-11-28T06:54:27,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/03ca83dcf1a94b78b28f09e120b2098f as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/03ca83dcf1a94b78b28f09e120b2098f 2024-11-28T06:54:27,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/03ca83dcf1a94b78b28f09e120b2098f, entries=150, sequenceid=103, filesize=30.2 K 2024-11-28T06:54:27,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/258f04dd2eb848ba8bf6d0accef72e87 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/258f04dd2eb848ba8bf6d0accef72e87 2024-11-28T06:54:27,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/258f04dd2eb848ba8bf6d0accef72e87, entries=150, sequenceid=103, filesize=11.7 K 2024-11-28T06:54:27,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/e4c757d6a8e1485294dc12e68f47d530 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/e4c757d6a8e1485294dc12e68f47d530 2024-11-28T06:54:27,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/e4c757d6a8e1485294dc12e68f47d530, entries=150, sequenceid=103, filesize=11.7 K 2024-11-28T06:54:27,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 5f1556cdc2c5439575ba4a9f6cd46c4e in 1681ms, sequenceid=103, compaction requested=true 2024-11-28T06:54:27,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: 2024-11-28T06:54:27,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1556cdc2c5439575ba4a9f6cd46c4e:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:54:27,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:27,493 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:54:27,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1556cdc2c5439575ba4a9f6cd46c4e:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:54:27,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:27,493 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:54:27,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1556cdc2c5439575ba4a9f6cd46c4e:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:54:27,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:54:27,494 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:54:27,494 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:54:27,494 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): 5f1556cdc2c5439575ba4a9f6cd46c4e/A is initiating minor compaction (all files) 2024-11-28T06:54:27,494 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): 5f1556cdc2c5439575ba4a9f6cd46c4e/B is initiating minor compaction (all files) 2024-11-28T06:54:27,494 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1556cdc2c5439575ba4a9f6cd46c4e/A in TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:27,494 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1556cdc2c5439575ba4a9f6cd46c4e/B in TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:27,494 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/d1d75e9b08024be497a6b0ad114d2362, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/1bb6e60b2f254e06927e2fee4eb7345f, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/03ca83dcf1a94b78b28f09e120b2098f] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp, totalSize=90.8 K 2024-11-28T06:54:27,494 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/8f36073e8c314e1481a7175a29c96855, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/4a3e83ff6f40448a9314cc9d05aba8f4, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/258f04dd2eb848ba8bf6d0accef72e87] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp, totalSize=35.3 K 2024-11-28T06:54:27,494 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:27,494 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. files: [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/d1d75e9b08024be497a6b0ad114d2362, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/1bb6e60b2f254e06927e2fee4eb7345f, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/03ca83dcf1a94b78b28f09e120b2098f] 2024-11-28T06:54:27,495 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f36073e8c314e1481a7175a29c96855, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732776864186 2024-11-28T06:54:27,495 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1d75e9b08024be497a6b0ad114d2362, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732776864186 2024-11-28T06:54:27,495 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a3e83ff6f40448a9314cc9d05aba8f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732776864300 2024-11-28T06:54:27,495 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bb6e60b2f254e06927e2fee4eb7345f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732776864300 2024-11-28T06:54:27,495 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 258f04dd2eb848ba8bf6d0accef72e87, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732776865470 2024-11-28T06:54:27,495 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03ca83dcf1a94b78b28f09e120b2098f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732776865470 2024-11-28T06:54:27,504 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:27,506 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1556cdc2c5439575ba4a9f6cd46c4e#B#compaction#123 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:54:27,507 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/80c55fb25fd7466bb2c808f3bc4e474e is 50, key is test_row_0/B:col10/1732776865809/Put/seqid=0 2024-11-28T06:54:27,509 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128972f8c2ab3a64bfe8a20edc0f315760d_5f1556cdc2c5439575ba4a9f6cd46c4e store=[table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:27,511 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128972f8c2ab3a64bfe8a20edc0f315760d_5f1556cdc2c5439575ba4a9f6cd46c4e, store=[table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:27,511 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128972f8c2ab3a64bfe8a20edc0f315760d_5f1556cdc2c5439575ba4a9f6cd46c4e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:27,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741969_1145 (size=4469) 2024-11-28T06:54:27,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-28T06:54:27,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:27,542 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 5f1556cdc2c5439575ba4a9f6cd46c4e 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T06:54:27,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=A 2024-11-28T06:54:27,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:27,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=B 2024-11-28T06:54:27,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:27,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=C 2024-11-28T06:54:27,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:27,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128c5656ec4d1c6453390a20db9b115b9f3_5f1556cdc2c5439575ba4a9f6cd46c4e is 50, key is test_row_0/A:col10/1732776865882/Put/seqid=0 2024-11-28T06:54:27,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741970_1146 (size=12154) 2024-11-28T06:54:27,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741968_1144 (size=12207) 2024-11-28T06:54:27,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-28T06:54:27,925 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1556cdc2c5439575ba4a9f6cd46c4e#A#compaction#122 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:54:27,926 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/19c4878db13d4dd7a68233f048b5c707 is 175, key is test_row_0/A:col10/1732776865809/Put/seqid=0 2024-11-28T06:54:27,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741971_1147 (size=31161) 2024-11-28T06:54:27,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:27,966 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/80c55fb25fd7466bb2c808f3bc4e474e as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/80c55fb25fd7466bb2c808f3bc4e474e 2024-11-28T06:54:27,967 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128c5656ec4d1c6453390a20db9b115b9f3_5f1556cdc2c5439575ba4a9f6cd46c4e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128c5656ec4d1c6453390a20db9b115b9f3_5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:27,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/63a8b70b6ad045469b7a45a62e7cdaea, store: [table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:27,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/63a8b70b6ad045469b7a45a62e7cdaea is 175, key is test_row_0/A:col10/1732776865882/Put/seqid=0 2024-11-28T06:54:27,974 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f1556cdc2c5439575ba4a9f6cd46c4e/B of 5f1556cdc2c5439575ba4a9f6cd46c4e into 80c55fb25fd7466bb2c808f3bc4e474e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:27,974 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: 2024-11-28T06:54:27,974 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., storeName=5f1556cdc2c5439575ba4a9f6cd46c4e/B, priority=13, startTime=1732776867493; duration=0sec 2024-11-28T06:54:27,974 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:54:27,974 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1556cdc2c5439575ba4a9f6cd46c4e:B 2024-11-28T06:54:27,975 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:54:27,976 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:54:27,976 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): 5f1556cdc2c5439575ba4a9f6cd46c4e/C is initiating minor compaction (all files) 2024-11-28T06:54:27,976 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1556cdc2c5439575ba4a9f6cd46c4e/C in TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:27,976 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/7fea1ff5423640d08f15382829855ff1, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/5a7de5b675d84610b0a00cb20cda66ee, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/e4c757d6a8e1485294dc12e68f47d530] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp, totalSize=35.3 K 2024-11-28T06:54:27,976 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fea1ff5423640d08f15382829855ff1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732776864186 2024-11-28T06:54:27,977 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a7de5b675d84610b0a00cb20cda66ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732776864300 2024-11-28T06:54:27,977 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting e4c757d6a8e1485294dc12e68f47d530, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732776865470 2024-11-28T06:54:27,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741972_1148 (size=30955) 2024-11-28T06:54:27,995 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1556cdc2c5439575ba4a9f6cd46c4e#C#compaction#125 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:54:27,996 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/4dcfb89cdc7d453eb6642db3dff2b29a is 50, key is test_row_0/C:col10/1732776865809/Put/seqid=0 2024-11-28T06:54:28,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741973_1149 (size=12207) 2024-11-28T06:54:28,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:28,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:28,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:28,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776928079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:28,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:28,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:28,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:28,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:28,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776928084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:28,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:28,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776928085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:28,094 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:28,094 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:28,094 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:28,095 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:28,095 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:28,095 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:28,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:28,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776928195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:28,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:28,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:28,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:28,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:28,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:28,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776928205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:28,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776928205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:28,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:28,209 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:28,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:28,209 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:28,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:28,209 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:28,336 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/19c4878db13d4dd7a68233f048b5c707 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/19c4878db13d4dd7a68233f048b5c707 2024-11-28T06:54:28,342 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f1556cdc2c5439575ba4a9f6cd46c4e/A of 5f1556cdc2c5439575ba4a9f6cd46c4e into 19c4878db13d4dd7a68233f048b5c707(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:28,342 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: 2024-11-28T06:54:28,342 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., storeName=5f1556cdc2c5439575ba4a9f6cd46c4e/A, priority=13, startTime=1732776867493; duration=0sec 2024-11-28T06:54:28,342 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:28,342 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1556cdc2c5439575ba4a9f6cd46c4e:A 2024-11-28T06:54:28,389 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=122, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/63a8b70b6ad045469b7a45a62e7cdaea 2024-11-28T06:54:28,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/f8e6a50afab6455d8626f09d248b8a9f is 50, key is test_row_0/B:col10/1732776865882/Put/seqid=0 2024-11-28T06:54:28,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:28,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776928406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:28,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:28,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:28,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:28,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:28,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776928415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:28,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:28,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776928415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:28,422 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/4dcfb89cdc7d453eb6642db3dff2b29a as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/4dcfb89cdc7d453eb6642db3dff2b29a 2024-11-28T06:54:28,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741974_1150 (size=12001) 2024-11-28T06:54:28,431 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f1556cdc2c5439575ba4a9f6cd46c4e/C of 5f1556cdc2c5439575ba4a9f6cd46c4e into 4dcfb89cdc7d453eb6642db3dff2b29a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:28,431 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: 2024-11-28T06:54:28,432 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., storeName=5f1556cdc2c5439575ba4a9f6cd46c4e/C, priority=13, startTime=1732776867493; duration=0sec 2024-11-28T06:54:28,432 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:28,432 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1556cdc2c5439575ba4a9f6cd46c4e:C 2024-11-28T06:54:28,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41430 deadline: 1732776928525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:28,526 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_0' in 5f1556cdc2c5439575ba4a9f6cd46c4e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:54:28,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:28,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:28,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:28,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:28,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41388 deadline: 1732776928538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:28,540 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_2' in 5f1556cdc2c5439575ba4a9f6cd46c4e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:54:28,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:28,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:28,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-28T06:54:28,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:28,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776928726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776928726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:28,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776928726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:28,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:28,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:28,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:28,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:28,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:28,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:28,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:28,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:28,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:28,829 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/f8e6a50afab6455d8626f09d248b8a9f 2024-11-28T06:54:28,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/f3df5e82941945b9bd58a1fe95c85ee4 is 50, key is test_row_0/C:col10/1732776865882/Put/seqid=0 2024-11-28T06:54:28,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741975_1151 (size=12001) 2024-11-28T06:54:29,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776929236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:29,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:29,236 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776929236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:29,236 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:29,236 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:29,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:29,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:29,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776929236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:29,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:29,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:29,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:29,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:29,250 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/f3df5e82941945b9bd58a1fe95c85ee4 2024-11-28T06:54:29,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/63a8b70b6ad045469b7a45a62e7cdaea as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/63a8b70b6ad045469b7a45a62e7cdaea 2024-11-28T06:54:29,297 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/63a8b70b6ad045469b7a45a62e7cdaea, entries=150, sequenceid=122, filesize=30.2 K 2024-11-28T06:54:29,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/f8e6a50afab6455d8626f09d248b8a9f as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/f8e6a50afab6455d8626f09d248b8a9f 2024-11-28T06:54:29,303 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/f8e6a50afab6455d8626f09d248b8a9f, entries=150, sequenceid=122, filesize=11.7 K 2024-11-28T06:54:29,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/f3df5e82941945b9bd58a1fe95c85ee4 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/f3df5e82941945b9bd58a1fe95c85ee4 2024-11-28T06:54:29,317 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/f3df5e82941945b9bd58a1fe95c85ee4, entries=150, sequenceid=122, filesize=11.7 K 2024-11-28T06:54:29,319 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 5f1556cdc2c5439575ba4a9f6cd46c4e in 1777ms, sequenceid=122, compaction requested=false 2024-11-28T06:54:29,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: 2024-11-28T06:54:29,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:29,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-28T06:54:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-28T06:54:29,323 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-28T06:54:29,323 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8510 sec 2024-11-28T06:54:29,325 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 2.8580 sec 2024-11-28T06:54:30,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:30,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f1556cdc2c5439575ba4a9f6cd46c4e 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-28T06:54:30,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=A 2024-11-28T06:54:30,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:30,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=B 2024-11-28T06:54:30,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:30,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=C 2024-11-28T06:54:30,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:30,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112810728cc5d08946588cdb2a40f3395440_5f1556cdc2c5439575ba4a9f6cd46c4e is 50, key is test_row_0/A:col10/1732776870249/Put/seqid=0 2024-11-28T06:54:30,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741976_1152 (size=14744) 2024-11-28T06:54:30,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:30,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776930275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:30,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:30,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:30,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:30,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:30,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776930276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:30,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:30,282 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:30,282 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:30,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776930276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:30,282 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:30,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:30,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:30,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:30,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:30,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776930385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:30,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:30,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:30,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:30,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776930385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:30,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:30,387 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:30,387 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:30,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:30,387 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:30,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776930385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:30,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:30,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:30,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:30,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:30,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776930595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:30,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:30,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:30,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776930595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:30,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:30,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:30,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:30,597 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:30,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776930596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:30,597 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:30,597 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:30,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:30,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:30,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:30,619 INFO [master/1cb9cf437a36:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-28T06:54:30,619 INFO [master/1cb9cf437a36:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-28T06:54:30,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-28T06:54:30,635 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:54:30,636 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:54:30,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-28T06:54:30,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-28T06:54:30,638 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:54:30,639 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:54:30,639 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:54:30,665 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:30,670 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112810728cc5d08946588cdb2a40f3395440_5f1556cdc2c5439575ba4a9f6cd46c4e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112810728cc5d08946588cdb2a40f3395440_5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:30,671 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/3198b8145f8b4757b42081553d3c8539, store: [table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:30,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/3198b8145f8b4757b42081553d3c8539 is 175, key is test_row_0/A:col10/1732776870249/Put/seqid=0 2024-11-28T06:54:30,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741977_1153 (size=39699) 2024-11-28T06:54:30,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-28T06:54:30,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-28T06:54:30,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:30,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:30,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:30,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:30,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:30,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:30,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:30,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776930905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:30,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:30,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:30,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:30,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776930905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:30,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:30,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:30,908 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:30,908 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:30,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776930905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:30,908 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:30,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:30,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:30,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:30,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-28T06:54:30,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:30,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:30,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:30,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:30,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:30,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:30,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-28T06:54:31,077 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=142, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/3198b8145f8b4757b42081553d3c8539 2024-11-28T06:54:31,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/6a032ef8cd4740f5a2d353cff25eac41 is 50, key is test_row_0/B:col10/1732776870249/Put/seqid=0 2024-11-28T06:54:31,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741978_1154 (size=12151) 2024-11-28T06:54:31,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/6a032ef8cd4740f5a2d353cff25eac41 2024-11-28T06:54:31,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-28T06:54:31,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:31,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:31,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:31,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:31,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:31,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/b0c0d36f3e8a4d718b8ff0fe082b0423 is 50, key is test_row_0/C:col10/1732776870249/Put/seqid=0 2024-11-28T06:54:31,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741979_1155 (size=12151) 2024-11-28T06:54:31,120 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/b0c0d36f3e8a4d718b8ff0fe082b0423 2024-11-28T06:54:31,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/3198b8145f8b4757b42081553d3c8539 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/3198b8145f8b4757b42081553d3c8539 2024-11-28T06:54:31,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/3198b8145f8b4757b42081553d3c8539, entries=200, sequenceid=142, filesize=38.8 K 2024-11-28T06:54:31,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/6a032ef8cd4740f5a2d353cff25eac41 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/6a032ef8cd4740f5a2d353cff25eac41 2024-11-28T06:54:31,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/6a032ef8cd4740f5a2d353cff25eac41, entries=150, sequenceid=142, filesize=11.9 K 2024-11-28T06:54:31,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/b0c0d36f3e8a4d718b8ff0fe082b0423 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/b0c0d36f3e8a4d718b8ff0fe082b0423 2024-11-28T06:54:31,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/b0c0d36f3e8a4d718b8ff0fe082b0423, entries=150, sequenceid=142, filesize=11.9 K 2024-11-28T06:54:31,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 5f1556cdc2c5439575ba4a9f6cd46c4e in 892ms, sequenceid=142, compaction requested=true 2024-11-28T06:54:31,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: 2024-11-28T06:54:31,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1556cdc2c5439575ba4a9f6cd46c4e:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:54:31,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:31,142 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:54:31,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1556cdc2c5439575ba4a9f6cd46c4e:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:54:31,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:31,142 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:54:31,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1556cdc2c5439575ba4a9f6cd46c4e:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:54:31,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:54:31,143 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101815 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:54:31,143 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): 5f1556cdc2c5439575ba4a9f6cd46c4e/A is initiating minor compaction (all files) 2024-11-28T06:54:31,143 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1556cdc2c5439575ba4a9f6cd46c4e/A in TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:31,144 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/19c4878db13d4dd7a68233f048b5c707, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/63a8b70b6ad045469b7a45a62e7cdaea, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/3198b8145f8b4757b42081553d3c8539] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp, totalSize=99.4 K 2024-11-28T06:54:31,144 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:31,144 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. files: [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/19c4878db13d4dd7a68233f048b5c707, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/63a8b70b6ad045469b7a45a62e7cdaea, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/3198b8145f8b4757b42081553d3c8539] 2024-11-28T06:54:31,144 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:54:31,144 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): 5f1556cdc2c5439575ba4a9f6cd46c4e/B is initiating minor compaction (all files) 2024-11-28T06:54:31,144 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1556cdc2c5439575ba4a9f6cd46c4e/B in TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:31,144 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/80c55fb25fd7466bb2c808f3bc4e474e, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/f8e6a50afab6455d8626f09d248b8a9f, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/6a032ef8cd4740f5a2d353cff25eac41] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp, totalSize=35.5 K 2024-11-28T06:54:31,146 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19c4878db13d4dd7a68233f048b5c707, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732776865470 2024-11-28T06:54:31,146 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 80c55fb25fd7466bb2c808f3bc4e474e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732776865470 2024-11-28T06:54:31,147 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting f8e6a50afab6455d8626f09d248b8a9f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732776865874 2024-11-28T06:54:31,147 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63a8b70b6ad045469b7a45a62e7cdaea, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732776865874 2024-11-28T06:54:31,148 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a032ef8cd4740f5a2d353cff25eac41, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732776868083 2024-11-28T06:54:31,148 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3198b8145f8b4757b42081553d3c8539, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732776868058 2024-11-28T06:54:31,156 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:31,161 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1556cdc2c5439575ba4a9f6cd46c4e#B#compaction#132 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:54:31,162 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/791ccbd82a5d476c840e9545152ac179 is 50, key is test_row_0/B:col10/1732776870249/Put/seqid=0 2024-11-28T06:54:31,164 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411286a3001c1d1bc41b3a4c05adb8201320e_5f1556cdc2c5439575ba4a9f6cd46c4e store=[table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:31,165 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411286a3001c1d1bc41b3a4c05adb8201320e_5f1556cdc2c5439575ba4a9f6cd46c4e, store=[table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:31,165 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411286a3001c1d1bc41b3a4c05adb8201320e_5f1556cdc2c5439575ba4a9f6cd46c4e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:31,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741980_1156 (size=12459) 2024-11-28T06:54:31,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741981_1157 (size=4469) 2024-11-28T06:54:31,187 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1556cdc2c5439575ba4a9f6cd46c4e#A#compaction#131 average throughput is 0.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:54:31,188 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/5762efb7bb164d58bc22f2a6d8db63c8 is 175, key is test_row_0/A:col10/1732776870249/Put/seqid=0 2024-11-28T06:54:31,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741982_1158 (size=31413) 2024-11-28T06:54:31,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-28T06:54:31,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:31,251 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 5f1556cdc2c5439575ba4a9f6cd46c4e 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-28T06:54:31,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=A 2024-11-28T06:54:31,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:31,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=B 2024-11-28T06:54:31,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:31,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=C 2024-11-28T06:54:31,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:31,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128b5c6c3682b9c4f15b4948735c354af1c_5f1556cdc2c5439575ba4a9f6cd46c4e is 50, key is test_row_0/A:col10/1732776870275/Put/seqid=0 2024-11-28T06:54:31,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741983_1159 (size=12304) 2024-11-28T06:54:31,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-28T06:54:31,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,275 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128b5c6c3682b9c4f15b4948735c354af1c_5f1556cdc2c5439575ba4a9f6cd46c4e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128b5c6c3682b9c4f15b4948735c354af1c_5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:31,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/dfe650a5a05a47fc8b125221c5d3b0c9, store: [table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:31,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/dfe650a5a05a47fc8b125221c5d3b0c9 is 175, key is test_row_0/A:col10/1732776870275/Put/seqid=0 2024-11-28T06:54:31,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741984_1160 (size=31105) 2024-11-28T06:54:31,293 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/dfe650a5a05a47fc8b125221c5d3b0c9 2024-11-28T06:54:31,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/5f3739986db54f4e8bb669f83ff8b4b9 is 50, key is test_row_0/B:col10/1732776870275/Put/seqid=0 2024-11-28T06:54:31,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741985_1161 (size=12151) 2024-11-28T06:54:31,337 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/5f3739986db54f4e8bb669f83ff8b4b9 2024-11-28T06:54:31,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/bac86d30ab85438da10d1264ee90c064 is 50, key is test_row_0/C:col10/1732776870275/Put/seqid=0 2024-11-28T06:54:31,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741986_1162 (size=12151) 2024-11-28T06:54:31,354 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/bac86d30ab85438da10d1264ee90c064 2024-11-28T06:54:31,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/dfe650a5a05a47fc8b125221c5d3b0c9 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/dfe650a5a05a47fc8b125221c5d3b0c9 2024-11-28T06:54:31,374 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/dfe650a5a05a47fc8b125221c5d3b0c9, entries=150, sequenceid=158, filesize=30.4 K 2024-11-28T06:54:31,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/5f3739986db54f4e8bb669f83ff8b4b9 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/5f3739986db54f4e8bb669f83ff8b4b9 2024-11-28T06:54:31,381 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/5f3739986db54f4e8bb669f83ff8b4b9, entries=150, sequenceid=158, filesize=11.9 K 2024-11-28T06:54:31,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/bac86d30ab85438da10d1264ee90c064 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/bac86d30ab85438da10d1264ee90c064 2024-11-28T06:54:31,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,388 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/bac86d30ab85438da10d1264ee90c064, entries=150, sequenceid=158, filesize=11.9 K 2024-11-28T06:54:31,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,389 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for 5f1556cdc2c5439575ba4a9f6cd46c4e in 138ms, sequenceid=158, compaction requested=true 2024-11-28T06:54:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: 2024-11-28T06:54:31,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:31,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-28T06:54:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-28T06:54:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,393 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-28T06:54:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,393 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 753 msec 2024-11-28T06:54:31,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,396 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 758 msec 2024-11-28T06:54:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:31,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f1556cdc2c5439575ba4a9f6cd46c4e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T06:54:31,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=A 2024-11-28T06:54:31,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:31,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=B 2024-11-28T06:54:31,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:31,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=C 2024-11-28T06:54:31,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128d6098472cf5e4261a0bc3d4a284f18ba_5f1556cdc2c5439575ba4a9f6cd46c4e is 50, key is test_row_0/A:col10/1732776871471/Put/seqid=0 2024-11-28T06:54:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741987_1163 (size=22268) 2024-11-28T06:54:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,495 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,500 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128d6098472cf5e4261a0bc3d4a284f18ba_5f1556cdc2c5439575ba4a9f6cd46c4e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d6098472cf5e4261a0bc3d4a284f18ba_5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,503 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/9b41700525d04b9686781c4d6fb0d062, store: [table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/9b41700525d04b9686781c4d6fb0d062 is 175, key is test_row_0/A:col10/1732776871471/Put/seqid=0 2024-11-28T06:54:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:31,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741988_1164 (size=65673) 2024-11-28T06:54:31,542 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732776871529, max timestamp=1732776871539Num uniques 859; 2024-11-28T06:54:31,542 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T06:54:31,545 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732776871529, max timestamp=1732776871539Num uniques 859; 2024-11-28T06:54:31,545 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T06:54:31,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732776871529, max timestamp=1732776871539Num uniques 859; 2024-11-28T06:54:31,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T06:54:31,583 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/791ccbd82a5d476c840e9545152ac179 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/791ccbd82a5d476c840e9545152ac179 2024-11-28T06:54:31,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776931578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:31,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776931579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:31,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776931583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:31,588 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f1556cdc2c5439575ba4a9f6cd46c4e/B of 5f1556cdc2c5439575ba4a9f6cd46c4e into 791ccbd82a5d476c840e9545152ac179(size=12.2 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:31,588 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: 2024-11-28T06:54:31,588 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., storeName=5f1556cdc2c5439575ba4a9f6cd46c4e/B, priority=13, startTime=1732776871142; duration=0sec 2024-11-28T06:54:31,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:31,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:31,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:31,589 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:54:31,589 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1556cdc2c5439575ba4a9f6cd46c4e:B 2024-11-28T06:54:31,589 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T06:54:31,591 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T06:54:31,591 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): 5f1556cdc2c5439575ba4a9f6cd46c4e/C is initiating minor compaction (all files) 2024-11-28T06:54:31,591 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1556cdc2c5439575ba4a9f6cd46c4e/C in TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:31,591 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/4dcfb89cdc7d453eb6642db3dff2b29a, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/f3df5e82941945b9bd58a1fe95c85ee4, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/b0c0d36f3e8a4d718b8ff0fe082b0423, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/bac86d30ab85438da10d1264ee90c064] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp, totalSize=47.4 K 2024-11-28T06:54:31,591 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 4dcfb89cdc7d453eb6642db3dff2b29a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732776865470 2024-11-28T06:54:31,592 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting f3df5e82941945b9bd58a1fe95c85ee4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732776865874 2024-11-28T06:54:31,592 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting b0c0d36f3e8a4d718b8ff0fe082b0423, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732776868083 2024-11-28T06:54:31,592 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting bac86d30ab85438da10d1264ee90c064, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732776870268 2024-11-28T06:54:31,601 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/5762efb7bb164d58bc22f2a6d8db63c8 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/5762efb7bb164d58bc22f2a6d8db63c8 2024-11-28T06:54:31,610 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f1556cdc2c5439575ba4a9f6cd46c4e/A of 5f1556cdc2c5439575ba4a9f6cd46c4e into 5762efb7bb164d58bc22f2a6d8db63c8(size=30.7 K), total size for store is 61.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:31,610 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: 2024-11-28T06:54:31,610 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., storeName=5f1556cdc2c5439575ba4a9f6cd46c4e/A, priority=13, startTime=1732776871142; duration=0sec 2024-11-28T06:54:31,610 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:31,610 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1556cdc2c5439575ba4a9f6cd46c4e:A 2024-11-28T06:54:31,612 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1556cdc2c5439575ba4a9f6cd46c4e#C#compaction#137 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:54:31,614 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/d8a85f4a59c9413296f93d08decc29db is 50, key is test_row_0/C:col10/1732776870275/Put/seqid=0 2024-11-28T06:54:31,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741989_1165 (size=12493) 2024-11-28T06:54:31,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:31,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776931695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:31,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:31,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776931695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:31,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:31,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:31,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:31,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:31,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:31,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:31,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776931695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:31,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:31,703 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:31,703 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:31,703 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:31,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-28T06:54:31,775 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:54:31,776 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:54:31,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-28T06:54:31,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-28T06:54:31,779 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:54:31,779 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:54:31,779 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:54:31,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-28T06:54:31,909 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/9b41700525d04b9686781c4d6fb0d062 2024-11-28T06:54:31,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:31,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776931905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:31,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:31,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:31,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776931905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:31,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:31,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:31,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:31,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:31,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:31,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:31,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776931905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:31,911 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:31,911 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:31,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:31,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/f6cabfd804cb4e16a4ffc20ccae3625e is 50, key is test_row_0/B:col10/1732776871471/Put/seqid=0 2024-11-28T06:54:31,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741990_1166 (size=12151) 2024-11-28T06:54:31,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-28T06:54:31,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:31,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:31,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:31,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:31,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:31,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:32,026 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/d8a85f4a59c9413296f93d08decc29db as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/d8a85f4a59c9413296f93d08decc29db 2024-11-28T06:54:32,030 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5f1556cdc2c5439575ba4a9f6cd46c4e/C of 5f1556cdc2c5439575ba4a9f6cd46c4e into d8a85f4a59c9413296f93d08decc29db(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:32,030 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: 2024-11-28T06:54:32,030 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., storeName=5f1556cdc2c5439575ba4a9f6cd46c4e/C, priority=12, startTime=1732776871142; duration=0sec 2024-11-28T06:54:32,030 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:32,030 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1556cdc2c5439575ba4a9f6cd46c4e:C 2024-11-28T06:54:32,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-28T06:54:32,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:32,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:32,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:32,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:32,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:32,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:32,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-28T06:54:32,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:32,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776932215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:32,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:32,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:32,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776932215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:32,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:32,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:32,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:32,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:32,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:32,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776932215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:32,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:32,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:32,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:32,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:32,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-28T06:54:32,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:32,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:32,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:32,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:32,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:32,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:32,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/f6cabfd804cb4e16a4ffc20ccae3625e 2024-11-28T06:54:32,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/9e47a8e99ff543d993cd02d4b3d16b9b is 50, key is test_row_0/C:col10/1732776871471/Put/seqid=0 2024-11-28T06:54:32,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741991_1167 (size=12151) 2024-11-28T06:54:32,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-28T06:54:32,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:32,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:32,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:32,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:32,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:32,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:32,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-28T06:54:32,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:32,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41430 deadline: 1732776932536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:32,542 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_0' in 5f1556cdc2c5439575ba4a9f6cd46c4e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:54:32,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:32,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:32,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:32,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-28T06:54:32,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:32,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:32,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:32,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:32,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:32,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:32,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:32,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41388 deadline: 1732776932555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:32,562 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_2' in 5f1556cdc2c5439575ba4a9f6cd46c4e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:54:32,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:32,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:32,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:32,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-28T06:54:32,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:32,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:32,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:32,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:32,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:32,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:32,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:32,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776932725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:32,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:32,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:32,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:32,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:32,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776932726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:32,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:32,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:32,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:32,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:32,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776932726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:32,734 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:32,734 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:32,734 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:32,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/9e47a8e99ff543d993cd02d4b3d16b9b 2024-11-28T06:54:32,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/9b41700525d04b9686781c4d6fb0d062 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/9b41700525d04b9686781c4d6fb0d062 2024-11-28T06:54:32,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/9b41700525d04b9686781c4d6fb0d062, entries=350, sequenceid=168, filesize=64.1 K 2024-11-28T06:54:32,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/f6cabfd804cb4e16a4ffc20ccae3625e as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/f6cabfd804cb4e16a4ffc20ccae3625e 2024-11-28T06:54:32,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/f6cabfd804cb4e16a4ffc20ccae3625e, entries=150, sequenceid=168, filesize=11.9 K 2024-11-28T06:54:32,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/9e47a8e99ff543d993cd02d4b3d16b9b as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/9e47a8e99ff543d993cd02d4b3d16b9b 2024-11-28T06:54:32,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/9e47a8e99ff543d993cd02d4b3d16b9b, entries=150, sequenceid=168, filesize=11.9 K 2024-11-28T06:54:32,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 5f1556cdc2c5439575ba4a9f6cd46c4e in 1281ms, sequenceid=168, compaction requested=true 2024-11-28T06:54:32,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: 2024-11-28T06:54:32,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1556cdc2c5439575ba4a9f6cd46c4e:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:54:32,764 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:54:32,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:32,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1556cdc2c5439575ba4a9f6cd46c4e:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:54:32,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:32,764 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:54:32,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1556cdc2c5439575ba4a9f6cd46c4e:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:54:32,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:54:32,765 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:54:32,765 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): 5f1556cdc2c5439575ba4a9f6cd46c4e/B is initiating minor compaction (all files) 2024-11-28T06:54:32,765 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1556cdc2c5439575ba4a9f6cd46c4e/B in TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:32,765 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/791ccbd82a5d476c840e9545152ac179, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/5f3739986db54f4e8bb669f83ff8b4b9, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/f6cabfd804cb4e16a4ffc20ccae3625e] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp, totalSize=35.9 K 2024-11-28T06:54:32,765 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 791ccbd82a5d476c840e9545152ac179, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732776868083 2024-11-28T06:54:32,766 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f3739986db54f4e8bb669f83ff8b4b9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732776870268 2024-11-28T06:54:32,766 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting f6cabfd804cb4e16a4ffc20ccae3625e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732776871471 2024-11-28T06:54:32,767 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128191 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:54:32,767 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): 5f1556cdc2c5439575ba4a9f6cd46c4e/A is initiating minor compaction (all files) 2024-11-28T06:54:32,767 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1556cdc2c5439575ba4a9f6cd46c4e/A in TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:32,767 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/5762efb7bb164d58bc22f2a6d8db63c8, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/dfe650a5a05a47fc8b125221c5d3b0c9, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/9b41700525d04b9686781c4d6fb0d062] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp, totalSize=125.2 K 2024-11-28T06:54:32,767 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:32,767 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. files: [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/5762efb7bb164d58bc22f2a6d8db63c8, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/dfe650a5a05a47fc8b125221c5d3b0c9, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/9b41700525d04b9686781c4d6fb0d062] 2024-11-28T06:54:32,767 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5762efb7bb164d58bc22f2a6d8db63c8, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732776868083 2024-11-28T06:54:32,768 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfe650a5a05a47fc8b125221c5d3b0c9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732776870268 2024-11-28T06:54:32,768 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b41700525d04b9686781c4d6fb0d062, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732776871442 2024-11-28T06:54:32,773 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1556cdc2c5439575ba4a9f6cd46c4e#B#compaction#140 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:54:32,773 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:32,773 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/2797ca822fc6404ab2457b246bb32751 is 50, key is test_row_0/B:col10/1732776871471/Put/seqid=0 2024-11-28T06:54:32,777 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128ce0b039e370242888c01921aba52e9cb_5f1556cdc2c5439575ba4a9f6cd46c4e store=[table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:32,780 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128ce0b039e370242888c01921aba52e9cb_5f1556cdc2c5439575ba4a9f6cd46c4e, store=[table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:32,780 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128ce0b039e370242888c01921aba52e9cb_5f1556cdc2c5439575ba4a9f6cd46c4e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:32,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741992_1168 (size=12561) 2024-11-28T06:54:32,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741993_1169 (size=4469) 2024-11-28T06:54:32,790 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1556cdc2c5439575ba4a9f6cd46c4e#A#compaction#141 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:54:32,790 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/768b2be978ea4193b291e530cea4db8a is 175, key is test_row_0/A:col10/1732776871471/Put/seqid=0 2024-11-28T06:54:32,792 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/2797ca822fc6404ab2457b246bb32751 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/2797ca822fc6404ab2457b246bb32751 2024-11-28T06:54:32,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741994_1170 (size=31515) 2024-11-28T06:54:32,799 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f1556cdc2c5439575ba4a9f6cd46c4e/B of 5f1556cdc2c5439575ba4a9f6cd46c4e into 2797ca822fc6404ab2457b246bb32751(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:32,799 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: 2024-11-28T06:54:32,799 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., storeName=5f1556cdc2c5439575ba4a9f6cd46c4e/B, priority=13, startTime=1732776872764; duration=0sec 2024-11-28T06:54:32,799 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:54:32,799 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1556cdc2c5439575ba4a9f6cd46c4e:B 2024-11-28T06:54:32,799 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-28T06:54:32,800 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-28T06:54:32,801 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-28T06:54:32,801 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. because compaction request was cancelled 2024-11-28T06:54:32,801 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1556cdc2c5439575ba4a9f6cd46c4e:C 2024-11-28T06:54:32,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-28T06:54:32,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:32,851 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 5f1556cdc2c5439575ba4a9f6cd46c4e 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-28T06:54:32,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=A 2024-11-28T06:54:32,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:54:32,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=B 2024-11-28T06:54:32,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:54:32,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=C 2024-11-28T06:54:32,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:54:32,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128d15cf7c887e94246aac48f9c7d8a5753_5f1556cdc2c5439575ba4a9f6cd46c4e is 50, key is test_row_0/A:col10/1732776871578/Put/seqid=0 2024-11-28T06:54:32,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741995_1171 (size=12304) 2024-11-28T06:54:32,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-28T06:54:33,211 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/768b2be978ea4193b291e530cea4db8a as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/768b2be978ea4193b291e530cea4db8a 2024-11-28T06:54:33,221 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f1556cdc2c5439575ba4a9f6cd46c4e/A of 5f1556cdc2c5439575ba4a9f6cd46c4e into 768b2be978ea4193b291e530cea4db8a(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:33,221 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: 2024-11-28T06:54:33,221 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., storeName=5f1556cdc2c5439575ba4a9f6cd46c4e/A, priority=13, startTime=1732776872763; duration=0sec 2024-11-28T06:54:33,222 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:33,222 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1556cdc2c5439575ba4a9f6cd46c4e:A 2024-11-28T06:54:33,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:33,285 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128d15cf7c887e94246aac48f9c7d8a5753_5f1556cdc2c5439575ba4a9f6cd46c4e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d15cf7c887e94246aac48f9c7d8a5753_5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:33,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/f413c1e97bfe49caaf772168333deb7d, store: [table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:33,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/f413c1e97bfe49caaf772168333deb7d is 175, key is test_row_0/A:col10/1732776871578/Put/seqid=0 2024-11-28T06:54:33,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741996_1172 (size=31105) 2024-11-28T06:54:33,697 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/f413c1e97bfe49caaf772168333deb7d 2024-11-28T06:54:33,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/36cd10a7d66b4818bc7751968624fcba is 50, key is test_row_0/B:col10/1732776871578/Put/seqid=0 2024-11-28T06:54:33,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741997_1173 (size=12151) 2024-11-28T06:54:33,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:33,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:33,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:33,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776933755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:33,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:33,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:33,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:33,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:33,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776933761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:33,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:33,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:33,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776933761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:33,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:33,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:33,764 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:33,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:33,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:33,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:33,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776933865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:33,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:33,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:33,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:33,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:33,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776933876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:33,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:33,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776933876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:33,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:33,880 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:33,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:33,880 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:33,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:33,880 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:33,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-28T06:54:34,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:34,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776934075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:34,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:34,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:34,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:34,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:34,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776934086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:34,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:34,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:34,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776934086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:34,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:34,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:34,090 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:34,090 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:34,090 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:34,113 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/36cd10a7d66b4818bc7751968624fcba 2024-11-28T06:54:34,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/855295f511d44b05b5ee5691dada5311 is 50, key is test_row_0/C:col10/1732776871578/Put/seqid=0 2024-11-28T06:54:34,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741998_1174 (size=12151) 2024-11-28T06:54:34,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:34,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776934385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:34,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:34,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:34,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:34,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:34,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776934395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:34,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:34,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:34,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776934395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:34,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:34,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:34,398 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:34,398 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:34,398 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:34,573 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/855295f511d44b05b5ee5691dada5311 2024-11-28T06:54:34,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/f413c1e97bfe49caaf772168333deb7d as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/f413c1e97bfe49caaf772168333deb7d 2024-11-28T06:54:34,582 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/f413c1e97bfe49caaf772168333deb7d, entries=150, sequenceid=207, filesize=30.4 K 2024-11-28T06:54:34,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/36cd10a7d66b4818bc7751968624fcba as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/36cd10a7d66b4818bc7751968624fcba 2024-11-28T06:54:34,588 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/36cd10a7d66b4818bc7751968624fcba, entries=150, sequenceid=207, filesize=11.9 K 2024-11-28T06:54:34,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/855295f511d44b05b5ee5691dada5311 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/855295f511d44b05b5ee5691dada5311 2024-11-28T06:54:34,593 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/855295f511d44b05b5ee5691dada5311, entries=150, sequenceid=207, filesize=11.9 K 2024-11-28T06:54:34,594 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=53.67 KB/54960 for 5f1556cdc2c5439575ba4a9f6cd46c4e in 1743ms, sequenceid=207, compaction requested=true 2024-11-28T06:54:34,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: 2024-11-28T06:54:34,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:34,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-28T06:54:34,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-28T06:54:34,597 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-28T06:54:34,597 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8160 sec 2024-11-28T06:54:34,599 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.8220 sec 2024-11-28T06:54:34,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:34,897 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f1556cdc2c5439575ba4a9f6cd46c4e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-28T06:54:34,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=A 2024-11-28T06:54:34,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:34,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=B 2024-11-28T06:54:34,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:34,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=C 2024-11-28T06:54:34,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:34,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128ecc237e99ba64fc4a27d5a8a094db0f5_5f1556cdc2c5439575ba4a9f6cd46c4e is 50, key is test_row_0/A:col10/1732776873759/Put/seqid=0 2024-11-28T06:54:34,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741999_1175 (size=14794) 2024-11-28T06:54:34,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:34,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776934928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:34,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:34,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776934929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:34,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:34,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:34,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:34,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:34,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:34,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:34,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:34,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776934931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:34,934 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:34,934 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:34,935 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:35,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:35,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776935036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:35,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:35,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:35,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:35,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:35,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776935045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:35,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:35,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776935045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:35,047 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:35,047 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:35,047 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:35,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:35,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:35,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:35,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:35,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776935246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:35,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:35,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:35,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:35,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:35,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776935255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:35,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:35,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:35,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:35,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:35,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776935255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:35,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:35,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:35,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:35,312 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:35,320 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128ecc237e99ba64fc4a27d5a8a094db0f5_5f1556cdc2c5439575ba4a9f6cd46c4e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128ecc237e99ba64fc4a27d5a8a094db0f5_5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:35,321 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/4fff2dbc672f4651a75ade667deaea61, store: [table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:35,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/4fff2dbc672f4651a75ade667deaea61 is 175, key is test_row_0/A:col10/1732776873759/Put/seqid=0 2024-11-28T06:54:35,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742000_1176 (size=39749) 2024-11-28T06:54:35,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:35,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776935555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:35,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:35,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:35,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:35,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:35,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776935565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:35,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:35,569 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:35,569 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:35,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776935566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:35,569 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:35,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:35,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:35,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:35,728 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=220, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/4fff2dbc672f4651a75ade667deaea61 2024-11-28T06:54:35,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/a1893b96ac734afea3ad5a080ddf10fb is 50, key is test_row_0/B:col10/1732776873759/Put/seqid=0 2024-11-28T06:54:35,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742001_1177 (size=12151) 2024-11-28T06:54:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-28T06:54:35,946 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:54:35,946 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:54:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-28T06:54:35,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-28T06:54:35,948 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:54:35,949 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:54:35,949 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:54:36,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-28T06:54:36,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:36,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776936066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:36,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:36,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:36,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:36,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:36,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:36,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776936076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:36,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776936076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:36,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:36,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:36,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:36,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:36,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:36,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:36,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-28T06:54:36,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:36,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:36,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:36,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:36,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:36,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:36,144 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/a1893b96ac734afea3ad5a080ddf10fb 2024-11-28T06:54:36,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/8b56cdd5dd2846a78bf0ab5842a46bc3 is 50, key is test_row_0/C:col10/1732776873759/Put/seqid=0 2024-11-28T06:54:36,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742002_1178 (size=12151) 2024-11-28T06:54:36,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-28T06:54:36,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:36,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:36,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:36,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:36,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:36,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:36,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-28T06:54:36,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-28T06:54:36,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:36,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:36,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:36,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:36,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:36,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:36,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/8b56cdd5dd2846a78bf0ab5842a46bc3 2024-11-28T06:54:36,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-28T06:54:36,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:36,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:36,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:36,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:36,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:36,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/4fff2dbc672f4651a75ade667deaea61 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/4fff2dbc672f4651a75ade667deaea61 2024-11-28T06:54:36,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:36,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/4fff2dbc672f4651a75ade667deaea61, entries=200, sequenceid=220, filesize=38.8 K 2024-11-28T06:54:36,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/a1893b96ac734afea3ad5a080ddf10fb as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/a1893b96ac734afea3ad5a080ddf10fb 2024-11-28T06:54:36,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/a1893b96ac734afea3ad5a080ddf10fb, entries=150, sequenceid=220, filesize=11.9 K 2024-11-28T06:54:36,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/8b56cdd5dd2846a78bf0ab5842a46bc3 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/8b56cdd5dd2846a78bf0ab5842a46bc3 2024-11-28T06:54:36,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-28T06:54:36,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/8b56cdd5dd2846a78bf0ab5842a46bc3, entries=150, sequenceid=220, filesize=11.9 K 2024-11-28T06:54:36,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 5f1556cdc2c5439575ba4a9f6cd46c4e in 1684ms, sequenceid=220, compaction requested=true 2024-11-28T06:54:36,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: 2024-11-28T06:54:36,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1556cdc2c5439575ba4a9f6cd46c4e:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:54:36,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:36,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1556cdc2c5439575ba4a9f6cd46c4e:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:54:36,581 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:54:36,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:54:36,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1556cdc2c5439575ba4a9f6cd46c4e:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:54:36,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-28T06:54:36,581 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T06:54:36,582 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:54:36,582 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): 5f1556cdc2c5439575ba4a9f6cd46c4e/A is initiating minor compaction (all files) 2024-11-28T06:54:36,582 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1556cdc2c5439575ba4a9f6cd46c4e/A in TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:36,582 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/768b2be978ea4193b291e530cea4db8a, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/f413c1e97bfe49caaf772168333deb7d, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/4fff2dbc672f4651a75ade667deaea61] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp, totalSize=100.0 K 2024-11-28T06:54:36,582 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:36,583 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. files: [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/768b2be978ea4193b291e530cea4db8a, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/f413c1e97bfe49caaf772168333deb7d, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/4fff2dbc672f4651a75ade667deaea61] 2024-11-28T06:54:36,583 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T06:54:36,583 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): 5f1556cdc2c5439575ba4a9f6cd46c4e/C is initiating minor compaction (all files) 2024-11-28T06:54:36,583 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 768b2be978ea4193b291e530cea4db8a, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732776871471 2024-11-28T06:54:36,583 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1556cdc2c5439575ba4a9f6cd46c4e/C in TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:36,583 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/d8a85f4a59c9413296f93d08decc29db, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/9e47a8e99ff543d993cd02d4b3d16b9b, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/855295f511d44b05b5ee5691dada5311, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/8b56cdd5dd2846a78bf0ab5842a46bc3] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp, totalSize=47.8 K 2024-11-28T06:54:36,583 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting f413c1e97bfe49caaf772168333deb7d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732776871576 2024-11-28T06:54:36,583 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting d8a85f4a59c9413296f93d08decc29db, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732776870268 2024-11-28T06:54:36,584 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fff2dbc672f4651a75ade667deaea61, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732776873748 2024-11-28T06:54:36,584 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e47a8e99ff543d993cd02d4b3d16b9b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732776871471 2024-11-28T06:54:36,584 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 855295f511d44b05b5ee5691dada5311, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732776871576 2024-11-28T06:54:36,584 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b56cdd5dd2846a78bf0ab5842a46bc3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732776873748 2024-11-28T06:54:36,591 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:36,592 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411285d7a6d426e1541828ab8b996be26d520_5f1556cdc2c5439575ba4a9f6cd46c4e store=[table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:36,594 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411285d7a6d426e1541828ab8b996be26d520_5f1556cdc2c5439575ba4a9f6cd46c4e, store=[table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:36,594 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411285d7a6d426e1541828ab8b996be26d520_5f1556cdc2c5439575ba4a9f6cd46c4e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:36,600 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1556cdc2c5439575ba4a9f6cd46c4e#C#compaction#149 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:54:36,600 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/d848f538d73041cfb90cd55dcd86d2be is 50, key is test_row_0/C:col10/1732776873759/Put/seqid=0 2024-11-28T06:54:36,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742003_1179 (size=4469) 2024-11-28T06:54:36,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742004_1180 (size=12629) 2024-11-28T06:54:36,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-28T06:54:36,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:36,712 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 5f1556cdc2c5439575ba4a9f6cd46c4e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T06:54:36,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=A 2024-11-28T06:54:36,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:36,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=B 2024-11-28T06:54:36,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:36,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=C 2024-11-28T06:54:36,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:36,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411285b132c11ab1e4a0985ddd6de87e99706_5f1556cdc2c5439575ba4a9f6cd46c4e is 50, key is test_row_0/A:col10/1732776874916/Put/seqid=0 2024-11-28T06:54:36,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742005_1181 (size=12304) 2024-11-28T06:54:37,013 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1556cdc2c5439575ba4a9f6cd46c4e#A#compaction#148 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:54:37,014 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/6ce1b2ccceaf43d3aec18cb3be5d260b is 175, key is test_row_0/A:col10/1732776873759/Put/seqid=0 2024-11-28T06:54:37,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742006_1182 (size=31617) 2024-11-28T06:54:37,034 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/d848f538d73041cfb90cd55dcd86d2be as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/d848f538d73041cfb90cd55dcd86d2be 2024-11-28T06:54:37,040 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5f1556cdc2c5439575ba4a9f6cd46c4e/C of 5f1556cdc2c5439575ba4a9f6cd46c4e into d848f538d73041cfb90cd55dcd86d2be(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:37,040 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: 2024-11-28T06:54:37,040 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., storeName=5f1556cdc2c5439575ba4a9f6cd46c4e/C, priority=12, startTime=1732776876581; duration=0sec 2024-11-28T06:54:37,040 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:54:37,040 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1556cdc2c5439575ba4a9f6cd46c4e:C 2024-11-28T06:54:37,040 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:54:37,042 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:54:37,042 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): 5f1556cdc2c5439575ba4a9f6cd46c4e/B is initiating minor compaction (all files) 2024-11-28T06:54:37,042 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1556cdc2c5439575ba4a9f6cd46c4e/B in TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:37,042 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/2797ca822fc6404ab2457b246bb32751, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/36cd10a7d66b4818bc7751968624fcba, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/a1893b96ac734afea3ad5a080ddf10fb] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp, totalSize=36.0 K 2024-11-28T06:54:37,042 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 2797ca822fc6404ab2457b246bb32751, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732776871471 2024-11-28T06:54:37,043 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 36cd10a7d66b4818bc7751968624fcba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732776871576 2024-11-28T06:54:37,043 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting a1893b96ac734afea3ad5a080ddf10fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732776873748 2024-11-28T06:54:37,051 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1556cdc2c5439575ba4a9f6cd46c4e#B#compaction#151 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:54:37,052 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/b667b4d0e74b42b1b6c6ffa6b94cc54b is 50, key is test_row_0/B:col10/1732776873759/Put/seqid=0 2024-11-28T06:54:37,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742007_1183 (size=12663) 2024-11-28T06:54:37,072 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/b667b4d0e74b42b1b6c6ffa6b94cc54b as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/b667b4d0e74b42b1b6c6ffa6b94cc54b 2024-11-28T06:54:37,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:37,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:37,078 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f1556cdc2c5439575ba4a9f6cd46c4e/B of 5f1556cdc2c5439575ba4a9f6cd46c4e into b667b4d0e74b42b1b6c6ffa6b94cc54b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:37,078 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: 2024-11-28T06:54:37,078 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., storeName=5f1556cdc2c5439575ba4a9f6cd46c4e/B, priority=13, startTime=1732776876581; duration=0sec 2024-11-28T06:54:37,078 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:37,078 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1556cdc2c5439575ba4a9f6cd46c4e:B 2024-11-28T06:54:37,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-28T06:54:37,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:37,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776937095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:37,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:37,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776937095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:37,102 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_0' in 5f1556cdc2c5439575ba4a9f6cd46c4e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:54:37,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:37,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:37,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:37,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:37,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776937096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:37,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:37,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:37,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:37,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:37,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:37,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:37,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:37,130 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411285b132c11ab1e4a0985ddd6de87e99706_5f1556cdc2c5439575ba4a9f6cd46c4e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411285b132c11ab1e4a0985ddd6de87e99706_5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:37,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/6bad9ce3a4794c61a08407dbeadd2613, store: [table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:37,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/6bad9ce3a4794c61a08407dbeadd2613 is 175, key is test_row_0/A:col10/1732776874916/Put/seqid=0 2024-11-28T06:54:37,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742008_1184 (size=31105) 2024-11-28T06:54:37,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:37,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776937205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:37,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:37,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:37,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:37,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:37,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776937215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:37,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:37,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:37,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:37,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:37,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776937415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:37,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:37,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:37,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:37,422 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/6ce1b2ccceaf43d3aec18cb3be5d260b as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/6ce1b2ccceaf43d3aec18cb3be5d260b 2024-11-28T06:54:37,427 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f1556cdc2c5439575ba4a9f6cd46c4e/A of 5f1556cdc2c5439575ba4a9f6cd46c4e into 6ce1b2ccceaf43d3aec18cb3be5d260b(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:37,427 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: 2024-11-28T06:54:37,427 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., storeName=5f1556cdc2c5439575ba4a9f6cd46c4e/A, priority=13, startTime=1732776876581; duration=0sec 2024-11-28T06:54:37,428 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:37,428 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1556cdc2c5439575ba4a9f6cd46c4e:A 2024-11-28T06:54:37,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:37,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776937425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:37,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:37,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:37,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:37,538 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=240, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/6bad9ce3a4794c61a08407dbeadd2613 2024-11-28T06:54:37,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/884389a1e884463aaf6756c621b1833a is 50, key is test_row_0/B:col10/1732776874916/Put/seqid=0 2024-11-28T06:54:37,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742009_1185 (size=12151) 2024-11-28T06:54:37,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:37,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776937725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:37,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:37,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:37,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:37,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:37,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776937736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:37,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:37,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:37,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:37,951 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/884389a1e884463aaf6756c621b1833a 2024-11-28T06:54:37,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/ac1c034039544870afd6c49b32f4bfc5 is 50, key is test_row_0/C:col10/1732776874916/Put/seqid=0 2024-11-28T06:54:37,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742010_1186 (size=12151) 2024-11-28T06:54:38,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-28T06:54:38,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:38,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776938236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:38,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:38,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:38,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:38,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:38,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776938246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:38,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:38,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:38,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:38,366 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/ac1c034039544870afd6c49b32f4bfc5 2024-11-28T06:54:38,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/6bad9ce3a4794c61a08407dbeadd2613 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/6bad9ce3a4794c61a08407dbeadd2613 2024-11-28T06:54:38,377 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/6bad9ce3a4794c61a08407dbeadd2613, entries=150, sequenceid=240, filesize=30.4 K 2024-11-28T06:54:38,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/884389a1e884463aaf6756c621b1833a as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/884389a1e884463aaf6756c621b1833a 2024-11-28T06:54:38,383 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/884389a1e884463aaf6756c621b1833a, entries=150, sequenceid=240, filesize=11.9 K 2024-11-28T06:54:38,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/ac1c034039544870afd6c49b32f4bfc5 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/ac1c034039544870afd6c49b32f4bfc5 2024-11-28T06:54:38,389 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/ac1c034039544870afd6c49b32f4bfc5, entries=150, sequenceid=240, filesize=11.9 K 2024-11-28T06:54:38,390 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 5f1556cdc2c5439575ba4a9f6cd46c4e in 1678ms, sequenceid=240, compaction requested=false 2024-11-28T06:54:38,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: 2024-11-28T06:54:38,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:38,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-28T06:54:38,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-28T06:54:38,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-28T06:54:38,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4420 sec 2024-11-28T06:54:38,394 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.4470 sec 2024-11-28T06:54:39,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:39,119 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f1556cdc2c5439575ba4a9f6cd46c4e 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T06:54:39,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=A 2024-11-28T06:54:39,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:39,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=B 2024-11-28T06:54:39,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:39,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=C 2024-11-28T06:54:39,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:39,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411282cdcdb627901483498a36c5e4496b33d_5f1556cdc2c5439575ba4a9f6cd46c4e is 50, key is test_row_0/A:col10/1732776879118/Put/seqid=0 2024-11-28T06:54:39,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742011_1187 (size=14794) 2024-11-28T06:54:39,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:39,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776939183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:39,186 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:39,186 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:39,186 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:39,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:39,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776939255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:39,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:39,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776939255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:39,258 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_2' in 5f1556cdc2c5439575ba4a9f6cd46c4e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:54:39,258 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_1' in 5f1556cdc2c5439575ba4a9f6cd46c4e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:54:39,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:39,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:39,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:39,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:39,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:39,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:39,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:39,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776939295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:39,298 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:39,298 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:39,298 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:39,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:39,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776939506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:39,507 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:39,507 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:39,507 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:39,534 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:39,573 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411282cdcdb627901483498a36c5e4496b33d_5f1556cdc2c5439575ba4a9f6cd46c4e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411282cdcdb627901483498a36c5e4496b33d_5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:39,574 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/7d5f6598eec149e3a0771493fdf7c7f8, store: [table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:39,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/7d5f6598eec149e3a0771493fdf7c7f8 is 175, key is test_row_0/A:col10/1732776879118/Put/seqid=0 2024-11-28T06:54:39,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742012_1188 (size=39749) 2024-11-28T06:54:39,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:39,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776939815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:39,818 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:39,818 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:39,818 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:39,979 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=256, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/7d5f6598eec149e3a0771493fdf7c7f8 2024-11-28T06:54:39,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/00fdef23b5484f19931d278975889ac5 is 50, key is test_row_0/B:col10/1732776879118/Put/seqid=0 2024-11-28T06:54:39,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742013_1189 (size=12151) 2024-11-28T06:54:40,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-28T06:54:40,126 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:54:40,126 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:54:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-28T06:54:40,128 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:54:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-28T06:54:40,129 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:54:40,129 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:54:40,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-28T06:54:40,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-28T06:54:40,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:40,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:40,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:40,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:40,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:40,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:40,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:40,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776940325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:40,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:40,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:40,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:40,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/00fdef23b5484f19931d278975889ac5 2024-11-28T06:54:40,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/44a3908bedaa47baabe612d84097b4f4 is 50, key is test_row_0/C:col10/1732776879118/Put/seqid=0 2024-11-28T06:54:40,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742014_1190 (size=12151) 2024-11-28T06:54:40,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-28T06:54:40,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:40,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:40,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:40,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:40,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:40,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:40,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-28T06:54:40,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-28T06:54:40,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:40,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:40,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:40,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:40,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:40,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:40,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-28T06:54:40,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:40,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:40,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:40,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:40,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:40,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:40,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-28T06:54:40,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/44a3908bedaa47baabe612d84097b4f4 2024-11-28T06:54:40,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/7d5f6598eec149e3a0771493fdf7c7f8 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/7d5f6598eec149e3a0771493fdf7c7f8 2024-11-28T06:54:40,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/7d5f6598eec149e3a0771493fdf7c7f8, entries=200, sequenceid=256, filesize=38.8 K 2024-11-28T06:54:40,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/00fdef23b5484f19931d278975889ac5 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/00fdef23b5484f19931d278975889ac5 2024-11-28T06:54:40,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/00fdef23b5484f19931d278975889ac5, entries=150, sequenceid=256, filesize=11.9 K 2024-11-28T06:54:40,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/44a3908bedaa47baabe612d84097b4f4 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/44a3908bedaa47baabe612d84097b4f4 2024-11-28T06:54:40,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/44a3908bedaa47baabe612d84097b4f4, entries=150, sequenceid=256, filesize=11.9 K 2024-11-28T06:54:40,828 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 5f1556cdc2c5439575ba4a9f6cd46c4e in 1709ms, sequenceid=256, compaction requested=true 2024-11-28T06:54:40,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: 2024-11-28T06:54:40,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1556cdc2c5439575ba4a9f6cd46c4e:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:54:40,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:40,828 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:54:40,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1556cdc2c5439575ba4a9f6cd46c4e:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:54:40,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:40,829 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:54:40,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1556cdc2c5439575ba4a9f6cd46c4e:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:54:40,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:54:40,829 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102471 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:54:40,829 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:54:40,829 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): 5f1556cdc2c5439575ba4a9f6cd46c4e/A is initiating minor compaction (all files) 2024-11-28T06:54:40,829 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): 5f1556cdc2c5439575ba4a9f6cd46c4e/B is initiating minor compaction (all files) 2024-11-28T06:54:40,830 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1556cdc2c5439575ba4a9f6cd46c4e/A in TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:40,830 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1556cdc2c5439575ba4a9f6cd46c4e/B in TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:40,830 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/6ce1b2ccceaf43d3aec18cb3be5d260b, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/6bad9ce3a4794c61a08407dbeadd2613, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/7d5f6598eec149e3a0771493fdf7c7f8] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp, totalSize=100.1 K 2024-11-28T06:54:40,830 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/b667b4d0e74b42b1b6c6ffa6b94cc54b, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/884389a1e884463aaf6756c621b1833a, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/00fdef23b5484f19931d278975889ac5] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp, totalSize=36.1 K 2024-11-28T06:54:40,830 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:40,830 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. files: [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/6ce1b2ccceaf43d3aec18cb3be5d260b, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/6bad9ce3a4794c61a08407dbeadd2613, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/7d5f6598eec149e3a0771493fdf7c7f8] 2024-11-28T06:54:40,830 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting b667b4d0e74b42b1b6c6ffa6b94cc54b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732776873748 2024-11-28T06:54:40,830 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ce1b2ccceaf43d3aec18cb3be5d260b, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732776873748 2024-11-28T06:54:40,831 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 884389a1e884463aaf6756c621b1833a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732776874916 2024-11-28T06:54:40,831 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bad9ce3a4794c61a08407dbeadd2613, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732776874916 2024-11-28T06:54:40,831 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 00fdef23b5484f19931d278975889ac5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732776877092 2024-11-28T06:54:40,831 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d5f6598eec149e3a0771493fdf7c7f8, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732776877090 2024-11-28T06:54:40,836 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:40,838 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1556cdc2c5439575ba4a9f6cd46c4e#B#compaction#158 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:54:40,839 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/7b76174de1d04a158873a258a295148c is 50, key is test_row_0/B:col10/1732776879118/Put/seqid=0 2024-11-28T06:54:40,843 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411285c2dc0379da643078833f0cf3dad324d_5f1556cdc2c5439575ba4a9f6cd46c4e store=[table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:40,845 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411285c2dc0379da643078833f0cf3dad324d_5f1556cdc2c5439575ba4a9f6cd46c4e, store=[table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:40,845 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411285c2dc0379da643078833f0cf3dad324d_5f1556cdc2c5439575ba4a9f6cd46c4e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:40,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742015_1191 (size=12765) 2024-11-28T06:54:40,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742016_1192 (size=4469) 2024-11-28T06:54:40,857 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/7b76174de1d04a158873a258a295148c as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/7b76174de1d04a158873a258a295148c 2024-11-28T06:54:40,862 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f1556cdc2c5439575ba4a9f6cd46c4e/B of 5f1556cdc2c5439575ba4a9f6cd46c4e into 7b76174de1d04a158873a258a295148c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:40,862 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: 2024-11-28T06:54:40,862 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., storeName=5f1556cdc2c5439575ba4a9f6cd46c4e/B, priority=13, startTime=1732776880828; duration=0sec 2024-11-28T06:54:40,862 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:54:40,862 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1556cdc2c5439575ba4a9f6cd46c4e:B 2024-11-28T06:54:40,862 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:54:40,864 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:54:40,864 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): 5f1556cdc2c5439575ba4a9f6cd46c4e/C is initiating minor compaction (all files) 2024-11-28T06:54:40,864 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1556cdc2c5439575ba4a9f6cd46c4e/C in TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:40,864 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/d848f538d73041cfb90cd55dcd86d2be, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/ac1c034039544870afd6c49b32f4bfc5, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/44a3908bedaa47baabe612d84097b4f4] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp, totalSize=36.1 K 2024-11-28T06:54:40,864 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting d848f538d73041cfb90cd55dcd86d2be, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732776873748 2024-11-28T06:54:40,864 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting ac1c034039544870afd6c49b32f4bfc5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732776874916 2024-11-28T06:54:40,865 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 44a3908bedaa47baabe612d84097b4f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732776877092 2024-11-28T06:54:40,872 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1556cdc2c5439575ba4a9f6cd46c4e#C#compaction#159 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:54:40,873 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/87be97f39a8b4a60ad046eb60278474f is 50, key is test_row_0/C:col10/1732776879118/Put/seqid=0 2024-11-28T06:54:40,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-28T06:54:40,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:40,893 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 5f1556cdc2c5439575ba4a9f6cd46c4e 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T06:54:40,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=A 2024-11-28T06:54:40,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:40,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=B 2024-11-28T06:54:40,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:40,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=C 2024-11-28T06:54:40,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:40,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742017_1193 (size=12731) 2024-11-28T06:54:40,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128faf7efb39f3e4dca964d7a8b80081c49_5f1556cdc2c5439575ba4a9f6cd46c4e is 50, key is test_row_0/A:col10/1732776879176/Put/seqid=0 2024-11-28T06:54:40,909 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/87be97f39a8b4a60ad046eb60278474f as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/87be97f39a8b4a60ad046eb60278474f 2024-11-28T06:54:40,919 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f1556cdc2c5439575ba4a9f6cd46c4e/C of 5f1556cdc2c5439575ba4a9f6cd46c4e into 87be97f39a8b4a60ad046eb60278474f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:40,919 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: 2024-11-28T06:54:40,919 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., storeName=5f1556cdc2c5439575ba4a9f6cd46c4e/C, priority=13, startTime=1732776880829; duration=0sec 2024-11-28T06:54:40,919 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:40,919 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1556cdc2c5439575ba4a9f6cd46c4e:C 2024-11-28T06:54:40,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742018_1194 (size=12454) 2024-11-28T06:54:41,257 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1556cdc2c5439575ba4a9f6cd46c4e#A#compaction#157 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:54:41,257 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/c10a45dc14044b17b6961bdd11454dff is 175, key is test_row_0/A:col10/1732776879118/Put/seqid=0 2024-11-28T06:54:41,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742019_1195 (size=31719) 2024-11-28T06:54:41,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-28T06:54:41,267 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/c10a45dc14044b17b6961bdd11454dff as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/c10a45dc14044b17b6961bdd11454dff 2024-11-28T06:54:41,272 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f1556cdc2c5439575ba4a9f6cd46c4e/A of 5f1556cdc2c5439575ba4a9f6cd46c4e into c10a45dc14044b17b6961bdd11454dff(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:41,272 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: 2024-11-28T06:54:41,272 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., storeName=5f1556cdc2c5439575ba4a9f6cd46c4e/A, priority=13, startTime=1732776880828; duration=0sec 2024-11-28T06:54:41,273 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:41,273 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1556cdc2c5439575ba4a9f6cd46c4e:A 2024-11-28T06:54:41,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:41,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. as already flushing 2024-11-28T06:54:41,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:41,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776941299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:41,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:41,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:41,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:41,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:41,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776941300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:41,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:41,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:41,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:41,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:41,325 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128faf7efb39f3e4dca964d7a8b80081c49_5f1556cdc2c5439575ba4a9f6cd46c4e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128faf7efb39f3e4dca964d7a8b80081c49_5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:41,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/5acf101cb8ad40deb2dba14b7e932e43, store: [table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:41,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/5acf101cb8ad40deb2dba14b7e932e43 is 175, key is test_row_0/A:col10/1732776879176/Put/seqid=0 2024-11-28T06:54:41,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742020_1196 (size=31255) 2024-11-28T06:54:41,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41362 deadline: 1732776941345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:41,347 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_0' in 5f1556cdc2c5439575ba4a9f6cd46c4e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:54:41,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:41,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:41,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:41,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:41,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776941416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:41,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:41,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:41,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:41,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:41,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776941416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:41,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:41,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:41,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:41,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:41,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776941625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:41,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:41,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:41,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:41,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776941625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:41,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:41,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:41,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:41,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:41,732 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=276, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/5acf101cb8ad40deb2dba14b7e932e43 2024-11-28T06:54:41,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/000f18f442634eb6a12c5e6d5b832ae4 is 50, key is test_row_0/B:col10/1732776879176/Put/seqid=0 2024-11-28T06:54:41,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742021_1197 (size=12301) 2024-11-28T06:54:41,841 INFO [Thread-618 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-618. 2024-11-28T06:54:41,842 DEBUG [Thread-618 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:54:41,842 DEBUG [Thread-618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:41,842 DEBUG [Thread-618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:41,842 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:54:41,843 INFO [Thread-622 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-622. 2024-11-28T06:54:41,843 DEBUG [Thread-622 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:54:41,843 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:41,844 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:41,844 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:54:41,846 INFO [Thread-620 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-620. 2024-11-28T06:54:41,846 DEBUG [Thread-620 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:54:41,846 DEBUG [Thread-620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:41,846 DEBUG [Thread-620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:41,846 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:54:41,851 INFO [Thread-624 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-624. 2024-11-28T06:54:41,852 DEBUG [Thread-624 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:54:41,852 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:41,852 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:41,852 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:54:41,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:41,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:41,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776941936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:41,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776941936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:41,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:41,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:41,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:41,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:41,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:41,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:42,152 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/000f18f442634eb6a12c5e6d5b832ae4 2024-11-28T06:54:42,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/d57e392536c64e7f883f191afc9e5bbb is 50, key is test_row_0/C:col10/1732776879176/Put/seqid=0 2024-11-28T06:54:42,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742022_1198 (size=12301) 2024-11-28T06:54:42,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-28T06:54:42,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:42,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:42,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41374 deadline: 1732776942445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:42,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41364 deadline: 1732776942445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:42,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:42,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:42,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:42,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:42,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:42,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:42,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:42,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41430 deadline: 1732776942566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:42,568 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_0' in 5f1556cdc2c5439575ba4a9f6cd46c4e of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18255 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:54:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:54:42,570 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/d57e392536c64e7f883f191afc9e5bbb 2024-11-28T06:54:42,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/5acf101cb8ad40deb2dba14b7e932e43 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/5acf101cb8ad40deb2dba14b7e932e43 2024-11-28T06:54:42,585 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/5acf101cb8ad40deb2dba14b7e932e43, entries=150, sequenceid=276, filesize=30.5 K 2024-11-28T06:54:42,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/000f18f442634eb6a12c5e6d5b832ae4 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/000f18f442634eb6a12c5e6d5b832ae4 2024-11-28T06:54:42,591 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/000f18f442634eb6a12c5e6d5b832ae4, entries=150, sequenceid=276, filesize=12.0 K 2024-11-28T06:54:42,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/d57e392536c64e7f883f191afc9e5bbb as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/d57e392536c64e7f883f191afc9e5bbb 2024-11-28T06:54:42,597 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/d57e392536c64e7f883f191afc9e5bbb, entries=150, sequenceid=276, filesize=12.0 K 2024-11-28T06:54:42,597 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 5f1556cdc2c5439575ba4a9f6cd46c4e in 1704ms, sequenceid=276, compaction requested=false 2024-11-28T06:54:42,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: 2024-11-28T06:54:42,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:42,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-28T06:54:42,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-28T06:54:42,600 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-28T06:54:42,600 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4700 sec 2024-11-28T06:54:42,602 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.4750 sec 2024-11-28T06:54:42,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:42,606 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f1556cdc2c5439575ba4a9f6cd46c4e 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T06:54:42,606 INFO [Thread-611 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-611. 2024-11-28T06:54:42,606 DEBUG [Thread-611 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:54:42,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=A 2024-11-28T06:54:42,607 DEBUG [Thread-611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:42,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:42,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=B 2024-11-28T06:54:42,607 DEBUG [Thread-611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:42,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:42,607 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:54:42,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=C 2024-11-28T06:54:42,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:42,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128eb619962091f44d5a31903b4c6d3e48c_5f1556cdc2c5439575ba4a9f6cd46c4e is 50, key is test_row_0/A:col10/1732776881298/Put/seqid=0 2024-11-28T06:54:42,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742023_1199 (size=12454) 2024-11-28T06:54:43,017 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:43,027 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128eb619962091f44d5a31903b4c6d3e48c_5f1556cdc2c5439575ba4a9f6cd46c4e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128eb619962091f44d5a31903b4c6d3e48c_5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:43,028 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/c9dfa74728d147f2b6ff50a4ff80bbd1, store: [table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:43,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/c9dfa74728d147f2b6ff50a4ff80bbd1 is 175, key is test_row_0/A:col10/1732776881298/Put/seqid=0 2024-11-28T06:54:43,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742024_1200 (size=31255) 2024-11-28T06:54:43,367 INFO [Thread-615 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-615. 2024-11-28T06:54:43,368 DEBUG [Thread-615 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:54:43,368 DEBUG [Thread-615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:43,368 DEBUG [Thread-615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:43,368 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:54:43,434 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=292, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/c9dfa74728d147f2b6ff50a4ff80bbd1 2024-11-28T06:54:43,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/986691564dc24dae8a818fc781a48161 is 50, key is test_row_0/B:col10/1732776881298/Put/seqid=0 2024-11-28T06:54:43,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742025_1201 (size=12301) 2024-11-28T06:54:43,458 INFO [Thread-613 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-613. 2024-11-28T06:54:43,458 INFO [Thread-607 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-607. 2024-11-28T06:54:43,458 DEBUG [Thread-613 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:54:43,458 DEBUG [Thread-607 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:54:43,458 DEBUG [Thread-613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:43,458 DEBUG [Thread-607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:43,458 DEBUG [Thread-613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:43,458 DEBUG [Thread-607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:43,458 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:54:43,459 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:54:43,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/986691564dc24dae8a818fc781a48161 2024-11-28T06:54:43,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/61fc73c2117a4670bcef1bb09f70d37d is 50, key is test_row_0/C:col10/1732776881298/Put/seqid=0 2024-11-28T06:54:43,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742026_1202 (size=12301) 2024-11-28T06:54:43,868 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T06:54:44,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/61fc73c2117a4670bcef1bb09f70d37d 2024-11-28T06:54:44,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/c9dfa74728d147f2b6ff50a4ff80bbd1 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/c9dfa74728d147f2b6ff50a4ff80bbd1 2024-11-28T06:54:44,283 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/c9dfa74728d147f2b6ff50a4ff80bbd1, entries=150, sequenceid=292, filesize=30.5 K 2024-11-28T06:54:44,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/986691564dc24dae8a818fc781a48161 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/986691564dc24dae8a818fc781a48161 2024-11-28T06:54:44,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/986691564dc24dae8a818fc781a48161, entries=150, sequenceid=292, filesize=12.0 K 2024-11-28T06:54:44,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/61fc73c2117a4670bcef1bb09f70d37d as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/61fc73c2117a4670bcef1bb09f70d37d 2024-11-28T06:54:44,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/61fc73c2117a4670bcef1bb09f70d37d, entries=150, sequenceid=292, filesize=12.0 K 2024-11-28T06:54:44,295 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=20.13 KB/20610 for 5f1556cdc2c5439575ba4a9f6cd46c4e in 1689ms, sequenceid=292, compaction requested=true 2024-11-28T06:54:44,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: 2024-11-28T06:54:44,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1556cdc2c5439575ba4a9f6cd46c4e:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:54:44,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:44,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1556cdc2c5439575ba4a9f6cd46c4e:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:54:44,295 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:54:44,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:44,295 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:54:44,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1556cdc2c5439575ba4a9f6cd46c4e:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:54:44,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:54:44,296 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:54:44,296 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:54:44,296 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): 5f1556cdc2c5439575ba4a9f6cd46c4e/A is initiating minor compaction (all files) 2024-11-28T06:54:44,296 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): 5f1556cdc2c5439575ba4a9f6cd46c4e/B is initiating minor compaction (all files) 2024-11-28T06:54:44,296 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1556cdc2c5439575ba4a9f6cd46c4e/A in TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:44,296 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1556cdc2c5439575ba4a9f6cd46c4e/B in TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:44,296 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/7b76174de1d04a158873a258a295148c, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/000f18f442634eb6a12c5e6d5b832ae4, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/986691564dc24dae8a818fc781a48161] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp, totalSize=36.5 K 2024-11-28T06:54:44,296 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/c10a45dc14044b17b6961bdd11454dff, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/5acf101cb8ad40deb2dba14b7e932e43, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/c9dfa74728d147f2b6ff50a4ff80bbd1] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp, totalSize=92.0 K 2024-11-28T06:54:44,296 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:44,296 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. files: [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/c10a45dc14044b17b6961bdd11454dff, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/5acf101cb8ad40deb2dba14b7e932e43, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/c9dfa74728d147f2b6ff50a4ff80bbd1] 2024-11-28T06:54:44,297 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b76174de1d04a158873a258a295148c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732776877092 2024-11-28T06:54:44,297 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting c10a45dc14044b17b6961bdd11454dff, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732776877092 2024-11-28T06:54:44,297 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 000f18f442634eb6a12c5e6d5b832ae4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732776879166 2024-11-28T06:54:44,297 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5acf101cb8ad40deb2dba14b7e932e43, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732776879166 2024-11-28T06:54:44,297 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 986691564dc24dae8a818fc781a48161, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732776881298 2024-11-28T06:54:44,298 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9dfa74728d147f2b6ff50a4ff80bbd1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732776881298 2024-11-28T06:54:44,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-28T06:54:44,305 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:54:44,306 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1556cdc2c5439575ba4a9f6cd46c4e#B#compaction#166 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:54:44,307 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/ea32228358d344ab95c190257d0b2354 is 50, key is test_row_0/B:col10/1732776881298/Put/seqid=0 2024-11-28T06:54:44,309 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:44,311 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411284d9cc2c1bc43463a802c42dfd85aa73f_5f1556cdc2c5439575ba4a9f6cd46c4e store=[table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:44,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742027_1203 (size=13017) 2024-11-28T06:54:44,342 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411284d9cc2c1bc43463a802c42dfd85aa73f_5f1556cdc2c5439575ba4a9f6cd46c4e, store=[table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:44,342 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411284d9cc2c1bc43463a802c42dfd85aa73f_5f1556cdc2c5439575ba4a9f6cd46c4e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:44,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742028_1204 (size=4469) 2024-11-28T06:54:44,747 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1556cdc2c5439575ba4a9f6cd46c4e#A#compaction#167 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:54:44,748 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/094f4a17e1754c9499457e0ba7c69e35 is 175, key is test_row_0/A:col10/1732776881298/Put/seqid=0 2024-11-28T06:54:44,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742029_1205 (size=31971) 2024-11-28T06:54:44,766 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/ea32228358d344ab95c190257d0b2354 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/ea32228358d344ab95c190257d0b2354 2024-11-28T06:54:44,772 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f1556cdc2c5439575ba4a9f6cd46c4e/B of 5f1556cdc2c5439575ba4a9f6cd46c4e into ea32228358d344ab95c190257d0b2354(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:44,772 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: 2024-11-28T06:54:44,772 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., storeName=5f1556cdc2c5439575ba4a9f6cd46c4e/B, priority=13, startTime=1732776884295; duration=0sec 2024-11-28T06:54:44,772 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:54:44,772 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1556cdc2c5439575ba4a9f6cd46c4e:B 2024-11-28T06:54:44,772 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:54:44,773 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:54:44,773 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): 5f1556cdc2c5439575ba4a9f6cd46c4e/C is initiating minor compaction (all files) 2024-11-28T06:54:44,773 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1556cdc2c5439575ba4a9f6cd46c4e/C in TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:44,774 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/87be97f39a8b4a60ad046eb60278474f, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/d57e392536c64e7f883f191afc9e5bbb, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/61fc73c2117a4670bcef1bb09f70d37d] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp, totalSize=36.5 K 2024-11-28T06:54:44,774 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 87be97f39a8b4a60ad046eb60278474f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732776877092 2024-11-28T06:54:44,774 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting d57e392536c64e7f883f191afc9e5bbb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732776879166 2024-11-28T06:54:44,775 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 61fc73c2117a4670bcef1bb09f70d37d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732776881298 2024-11-28T06:54:44,783 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1556cdc2c5439575ba4a9f6cd46c4e#C#compaction#168 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:54:44,784 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/526f6a35ccf4416cafc5c3bddfaa0794 is 50, key is test_row_0/C:col10/1732776881298/Put/seqid=0 2024-11-28T06:54:44,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742030_1206 (size=12983) 2024-11-28T06:54:45,173 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/094f4a17e1754c9499457e0ba7c69e35 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/094f4a17e1754c9499457e0ba7c69e35 2024-11-28T06:54:45,180 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f1556cdc2c5439575ba4a9f6cd46c4e/A of 5f1556cdc2c5439575ba4a9f6cd46c4e into 094f4a17e1754c9499457e0ba7c69e35(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:45,180 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: 2024-11-28T06:54:45,180 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., storeName=5f1556cdc2c5439575ba4a9f6cd46c4e/A, priority=13, startTime=1732776884295; duration=0sec 2024-11-28T06:54:45,180 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:45,180 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1556cdc2c5439575ba4a9f6cd46c4e:A 2024-11-28T06:54:45,192 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/526f6a35ccf4416cafc5c3bddfaa0794 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/526f6a35ccf4416cafc5c3bddfaa0794 2024-11-28T06:54:45,197 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f1556cdc2c5439575ba4a9f6cd46c4e/C of 5f1556cdc2c5439575ba4a9f6cd46c4e into 526f6a35ccf4416cafc5c3bddfaa0794(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:54:45,197 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: 2024-11-28T06:54:45,197 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., storeName=5f1556cdc2c5439575ba4a9f6cd46c4e/C, priority=13, startTime=1732776884295; duration=0sec 2024-11-28T06:54:45,197 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:54:45,197 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1556cdc2c5439575ba4a9f6cd46c4e:C 2024-11-28T06:54:52,651 INFO [Thread-609 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-609. 2024-11-28T06:54:52,651 DEBUG [Thread-609 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:54:52,651 DEBUG [Thread-609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:52,652 DEBUG [Thread-609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:52,652 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:54:52,652 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-28T06:54:52,652 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-28T06:54:52,653 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-28T06:54:52,653 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-28T06:54:52,653 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-28T06:54:52,653 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-28T06:54:52,653 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-28T06:54:52,653 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3454 2024-11-28T06:54:52,653 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3080 2024-11-28T06:54:52,653 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-28T06:54:52,653 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1350 2024-11-28T06:54:52,653 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4048 rows 2024-11-28T06:54:52,653 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1343 2024-11-28T06:54:52,653 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4027 rows 2024-11-28T06:54:52,654 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T06:54:52,654 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T06:54:52,654 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:52,654 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:52,654 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:54:52,656 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-28T06:54:52,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-28T06:54:52,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-28T06:54:52,662 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732776892661"}]},"ts":"1732776892661"} 2024-11-28T06:54:52,664 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-28T06:54:52,664 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-28T06:54:52,665 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T06:54:52,667 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f1556cdc2c5439575ba4a9f6cd46c4e, UNASSIGN}] 2024-11-28T06:54:52,668 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f1556cdc2c5439575ba4a9f6cd46c4e, UNASSIGN 2024-11-28T06:54:52,669 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=50 updating hbase:meta row=5f1556cdc2c5439575ba4a9f6cd46c4e, regionState=CLOSING, regionLocation=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:52,671 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f1556cdc2c5439575ba4a9f6cd46c4e, UNASSIGN because future has completed 2024-11-28T06:54:52,672 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T06:54:52,672 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568}] 2024-11-28T06:54:52,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-28T06:54:52,825 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(122): Close 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:52,826 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T06:54:52,826 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1722): Closing 5f1556cdc2c5439575ba4a9f6cd46c4e, disabling compactions & flushes 2024-11-28T06:54:52,826 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:52,826 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:52,826 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. after waiting 0 ms 2024-11-28T06:54:52,826 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:52,827 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(2902): Flushing 5f1556cdc2c5439575ba4a9f6cd46c4e 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-28T06:54:52,827 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=A 2024-11-28T06:54:52,827 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:52,827 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=B 2024-11-28T06:54:52,828 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:52,828 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f1556cdc2c5439575ba4a9f6cd46c4e, store=C 2024-11-28T06:54:52,828 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:52,838 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411280512ef973c9c46fe97ad060bbd86a2c8_5f1556cdc2c5439575ba4a9f6cd46c4e is 50, key is test_row_0/A:col10/1732776892647/Put/seqid=0 2024-11-28T06:54:52,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742031_1207 (size=12454) 2024-11-28T06:54:52,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-28T06:54:53,244 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:53,253 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411280512ef973c9c46fe97ad060bbd86a2c8_5f1556cdc2c5439575ba4a9f6cd46c4e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411280512ef973c9c46fe97ad060bbd86a2c8_5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:53,255 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/5acf9360d55e4c5a8d7fc44efdbb7726, store: [table=TestAcidGuarantees family=A region=5f1556cdc2c5439575ba4a9f6cd46c4e] 2024-11-28T06:54:53,255 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/5acf9360d55e4c5a8d7fc44efdbb7726 is 175, key is test_row_0/A:col10/1732776892647/Put/seqid=0 2024-11-28T06:54:53,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742032_1208 (size=31255) 2024-11-28T06:54:53,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-28T06:54:53,662 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=302, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/5acf9360d55e4c5a8d7fc44efdbb7726 2024-11-28T06:54:53,675 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/cacd9c99bbf04e2daa8fb25f15e585a1 is 50, key is test_row_0/B:col10/1732776892647/Put/seqid=0 2024-11-28T06:54:53,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742033_1209 (size=12301) 2024-11-28T06:54:53,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-28T06:54:54,080 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/cacd9c99bbf04e2daa8fb25f15e585a1 2024-11-28T06:54:54,095 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/3462c54d5be44e94a2753da0eb7b97c8 is 50, key is test_row_0/C:col10/1732776892647/Put/seqid=0 2024-11-28T06:54:54,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742034_1210 (size=12301) 2024-11-28T06:54:54,499 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/3462c54d5be44e94a2753da0eb7b97c8 2024-11-28T06:54:54,510 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/A/5acf9360d55e4c5a8d7fc44efdbb7726 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/5acf9360d55e4c5a8d7fc44efdbb7726 2024-11-28T06:54:54,517 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/5acf9360d55e4c5a8d7fc44efdbb7726, entries=150, sequenceid=302, filesize=30.5 K 2024-11-28T06:54:54,518 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/B/cacd9c99bbf04e2daa8fb25f15e585a1 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/cacd9c99bbf04e2daa8fb25f15e585a1 2024-11-28T06:54:54,522 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/cacd9c99bbf04e2daa8fb25f15e585a1, entries=150, sequenceid=302, filesize=12.0 K 2024-11-28T06:54:54,523 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/.tmp/C/3462c54d5be44e94a2753da0eb7b97c8 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/3462c54d5be44e94a2753da0eb7b97c8 2024-11-28T06:54:54,527 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/3462c54d5be44e94a2753da0eb7b97c8, entries=150, sequenceid=302, filesize=12.0 K 2024-11-28T06:54:54,528 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 5f1556cdc2c5439575ba4a9f6cd46c4e in 1702ms, sequenceid=302, compaction requested=false 2024-11-28T06:54:54,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/dd04892fa1bb46f78edc42b7eea22fc5, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/a6a388b5cffb4be380b242043353f613, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/0cb5aad427f041519f56376cf53ed968, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/d1d75e9b08024be497a6b0ad114d2362, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/1bb6e60b2f254e06927e2fee4eb7345f, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/19c4878db13d4dd7a68233f048b5c707, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/03ca83dcf1a94b78b28f09e120b2098f, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/63a8b70b6ad045469b7a45a62e7cdaea, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/3198b8145f8b4757b42081553d3c8539, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/5762efb7bb164d58bc22f2a6d8db63c8, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/dfe650a5a05a47fc8b125221c5d3b0c9, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/9b41700525d04b9686781c4d6fb0d062, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/768b2be978ea4193b291e530cea4db8a, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/f413c1e97bfe49caaf772168333deb7d, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/4fff2dbc672f4651a75ade667deaea61, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/6ce1b2ccceaf43d3aec18cb3be5d260b, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/6bad9ce3a4794c61a08407dbeadd2613, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/7d5f6598eec149e3a0771493fdf7c7f8, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/c10a45dc14044b17b6961bdd11454dff, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/5acf101cb8ad40deb2dba14b7e932e43, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/c9dfa74728d147f2b6ff50a4ff80bbd1] to archive 2024-11-28T06:54:54,529 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T06:54:54,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/dd04892fa1bb46f78edc42b7eea22fc5 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/dd04892fa1bb46f78edc42b7eea22fc5 2024-11-28T06:54:54,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/a6a388b5cffb4be380b242043353f613 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/a6a388b5cffb4be380b242043353f613 2024-11-28T06:54:54,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/0cb5aad427f041519f56376cf53ed968 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/0cb5aad427f041519f56376cf53ed968 2024-11-28T06:54:54,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/d1d75e9b08024be497a6b0ad114d2362 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/d1d75e9b08024be497a6b0ad114d2362 2024-11-28T06:54:54,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/1bb6e60b2f254e06927e2fee4eb7345f to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/1bb6e60b2f254e06927e2fee4eb7345f 2024-11-28T06:54:54,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/19c4878db13d4dd7a68233f048b5c707 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/19c4878db13d4dd7a68233f048b5c707 2024-11-28T06:54:54,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/03ca83dcf1a94b78b28f09e120b2098f to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/03ca83dcf1a94b78b28f09e120b2098f 2024-11-28T06:54:54,542 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/63a8b70b6ad045469b7a45a62e7cdaea to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/63a8b70b6ad045469b7a45a62e7cdaea 2024-11-28T06:54:54,543 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/3198b8145f8b4757b42081553d3c8539 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/3198b8145f8b4757b42081553d3c8539 2024-11-28T06:54:54,544 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/5762efb7bb164d58bc22f2a6d8db63c8 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/5762efb7bb164d58bc22f2a6d8db63c8 2024-11-28T06:54:54,545 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/dfe650a5a05a47fc8b125221c5d3b0c9 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/dfe650a5a05a47fc8b125221c5d3b0c9 2024-11-28T06:54:54,547 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/9b41700525d04b9686781c4d6fb0d062 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/9b41700525d04b9686781c4d6fb0d062 2024-11-28T06:54:54,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/768b2be978ea4193b291e530cea4db8a to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/768b2be978ea4193b291e530cea4db8a 2024-11-28T06:54:54,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/f413c1e97bfe49caaf772168333deb7d to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/f413c1e97bfe49caaf772168333deb7d 2024-11-28T06:54:54,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/4fff2dbc672f4651a75ade667deaea61 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/4fff2dbc672f4651a75ade667deaea61 2024-11-28T06:54:54,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/6ce1b2ccceaf43d3aec18cb3be5d260b to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/6ce1b2ccceaf43d3aec18cb3be5d260b 2024-11-28T06:54:54,552 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/6bad9ce3a4794c61a08407dbeadd2613 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/6bad9ce3a4794c61a08407dbeadd2613 2024-11-28T06:54:54,553 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/7d5f6598eec149e3a0771493fdf7c7f8 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/7d5f6598eec149e3a0771493fdf7c7f8 2024-11-28T06:54:54,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/c10a45dc14044b17b6961bdd11454dff to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/c10a45dc14044b17b6961bdd11454dff 2024-11-28T06:54:54,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/5acf101cb8ad40deb2dba14b7e932e43 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/5acf101cb8ad40deb2dba14b7e932e43 2024-11-28T06:54:54,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/c9dfa74728d147f2b6ff50a4ff80bbd1 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/c9dfa74728d147f2b6ff50a4ff80bbd1 2024-11-28T06:54:54,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/b75accbaf9ed4e8c9c5ae82d606e0536, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/0224219bb6944d8080cc383a28ad2391, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/8f36073e8c314e1481a7175a29c96855, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/5d3e83500b684d8fad7fd8d70c9816ca, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/4a3e83ff6f40448a9314cc9d05aba8f4, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/80c55fb25fd7466bb2c808f3bc4e474e, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/258f04dd2eb848ba8bf6d0accef72e87, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/f8e6a50afab6455d8626f09d248b8a9f, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/791ccbd82a5d476c840e9545152ac179, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/6a032ef8cd4740f5a2d353cff25eac41, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/5f3739986db54f4e8bb669f83ff8b4b9, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/2797ca822fc6404ab2457b246bb32751, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/f6cabfd804cb4e16a4ffc20ccae3625e, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/36cd10a7d66b4818bc7751968624fcba, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/b667b4d0e74b42b1b6c6ffa6b94cc54b, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/a1893b96ac734afea3ad5a080ddf10fb, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/884389a1e884463aaf6756c621b1833a, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/7b76174de1d04a158873a258a295148c, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/00fdef23b5484f19931d278975889ac5, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/000f18f442634eb6a12c5e6d5b832ae4, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/986691564dc24dae8a818fc781a48161] to archive 2024-11-28T06:54:54,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T06:54:54,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/b75accbaf9ed4e8c9c5ae82d606e0536 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/b75accbaf9ed4e8c9c5ae82d606e0536 2024-11-28T06:54:54,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/0224219bb6944d8080cc383a28ad2391 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/0224219bb6944d8080cc383a28ad2391 2024-11-28T06:54:54,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/8f36073e8c314e1481a7175a29c96855 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/8f36073e8c314e1481a7175a29c96855 2024-11-28T06:54:54,564 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/5d3e83500b684d8fad7fd8d70c9816ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/5d3e83500b684d8fad7fd8d70c9816ca 2024-11-28T06:54:54,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/4a3e83ff6f40448a9314cc9d05aba8f4 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/4a3e83ff6f40448a9314cc9d05aba8f4 2024-11-28T06:54:54,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/80c55fb25fd7466bb2c808f3bc4e474e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/80c55fb25fd7466bb2c808f3bc4e474e 2024-11-28T06:54:54,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/258f04dd2eb848ba8bf6d0accef72e87 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/258f04dd2eb848ba8bf6d0accef72e87 2024-11-28T06:54:54,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/f8e6a50afab6455d8626f09d248b8a9f to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/f8e6a50afab6455d8626f09d248b8a9f 2024-11-28T06:54:54,570 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/791ccbd82a5d476c840e9545152ac179 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/791ccbd82a5d476c840e9545152ac179 2024-11-28T06:54:54,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/6a032ef8cd4740f5a2d353cff25eac41 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/6a032ef8cd4740f5a2d353cff25eac41 2024-11-28T06:54:54,572 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/5f3739986db54f4e8bb669f83ff8b4b9 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/5f3739986db54f4e8bb669f83ff8b4b9 2024-11-28T06:54:54,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/2797ca822fc6404ab2457b246bb32751 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/2797ca822fc6404ab2457b246bb32751 2024-11-28T06:54:54,574 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/f6cabfd804cb4e16a4ffc20ccae3625e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/f6cabfd804cb4e16a4ffc20ccae3625e 2024-11-28T06:54:54,575 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/36cd10a7d66b4818bc7751968624fcba to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/36cd10a7d66b4818bc7751968624fcba 2024-11-28T06:54:54,576 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/b667b4d0e74b42b1b6c6ffa6b94cc54b to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/b667b4d0e74b42b1b6c6ffa6b94cc54b 2024-11-28T06:54:54,577 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/a1893b96ac734afea3ad5a080ddf10fb to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/a1893b96ac734afea3ad5a080ddf10fb 2024-11-28T06:54:54,578 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/884389a1e884463aaf6756c621b1833a to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/884389a1e884463aaf6756c621b1833a 2024-11-28T06:54:54,579 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/7b76174de1d04a158873a258a295148c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/7b76174de1d04a158873a258a295148c 2024-11-28T06:54:54,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/00fdef23b5484f19931d278975889ac5 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/00fdef23b5484f19931d278975889ac5 2024-11-28T06:54:54,581 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/000f18f442634eb6a12c5e6d5b832ae4 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/000f18f442634eb6a12c5e6d5b832ae4 2024-11-28T06:54:54,582 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/986691564dc24dae8a818fc781a48161 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/986691564dc24dae8a818fc781a48161 2024-11-28T06:54:54,583 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/d73c3d060ff54a0eb1277fe80feb4ed9, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/dc288fbf325d44c5ab100bd149120dc4, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/7fea1ff5423640d08f15382829855ff1, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/9573ee658a994b79b8934b19fdfa25c0, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/5a7de5b675d84610b0a00cb20cda66ee, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/4dcfb89cdc7d453eb6642db3dff2b29a, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/e4c757d6a8e1485294dc12e68f47d530, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/f3df5e82941945b9bd58a1fe95c85ee4, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/b0c0d36f3e8a4d718b8ff0fe082b0423, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/d8a85f4a59c9413296f93d08decc29db, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/bac86d30ab85438da10d1264ee90c064, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/9e47a8e99ff543d993cd02d4b3d16b9b, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/855295f511d44b05b5ee5691dada5311, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/d848f538d73041cfb90cd55dcd86d2be, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/8b56cdd5dd2846a78bf0ab5842a46bc3, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/ac1c034039544870afd6c49b32f4bfc5, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/87be97f39a8b4a60ad046eb60278474f, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/44a3908bedaa47baabe612d84097b4f4, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/d57e392536c64e7f883f191afc9e5bbb, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/61fc73c2117a4670bcef1bb09f70d37d] to archive 2024-11-28T06:54:54,584 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T06:54:54,585 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/d73c3d060ff54a0eb1277fe80feb4ed9 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/d73c3d060ff54a0eb1277fe80feb4ed9 2024-11-28T06:54:54,586 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/dc288fbf325d44c5ab100bd149120dc4 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/dc288fbf325d44c5ab100bd149120dc4 2024-11-28T06:54:54,587 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/7fea1ff5423640d08f15382829855ff1 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/7fea1ff5423640d08f15382829855ff1 2024-11-28T06:54:54,588 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/9573ee658a994b79b8934b19fdfa25c0 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/9573ee658a994b79b8934b19fdfa25c0 2024-11-28T06:54:54,588 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/5a7de5b675d84610b0a00cb20cda66ee to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/5a7de5b675d84610b0a00cb20cda66ee 2024-11-28T06:54:54,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/4dcfb89cdc7d453eb6642db3dff2b29a to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/4dcfb89cdc7d453eb6642db3dff2b29a 2024-11-28T06:54:54,590 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/e4c757d6a8e1485294dc12e68f47d530 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/e4c757d6a8e1485294dc12e68f47d530 2024-11-28T06:54:54,591 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/f3df5e82941945b9bd58a1fe95c85ee4 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/f3df5e82941945b9bd58a1fe95c85ee4 2024-11-28T06:54:54,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/b0c0d36f3e8a4d718b8ff0fe082b0423 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/b0c0d36f3e8a4d718b8ff0fe082b0423 2024-11-28T06:54:54,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/d8a85f4a59c9413296f93d08decc29db to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/d8a85f4a59c9413296f93d08decc29db 2024-11-28T06:54:54,593 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/bac86d30ab85438da10d1264ee90c064 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/bac86d30ab85438da10d1264ee90c064 2024-11-28T06:54:54,594 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/9e47a8e99ff543d993cd02d4b3d16b9b to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/9e47a8e99ff543d993cd02d4b3d16b9b 2024-11-28T06:54:54,594 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/855295f511d44b05b5ee5691dada5311 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/855295f511d44b05b5ee5691dada5311 2024-11-28T06:54:54,595 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/d848f538d73041cfb90cd55dcd86d2be to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/d848f538d73041cfb90cd55dcd86d2be 2024-11-28T06:54:54,596 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/8b56cdd5dd2846a78bf0ab5842a46bc3 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/8b56cdd5dd2846a78bf0ab5842a46bc3 2024-11-28T06:54:54,597 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/ac1c034039544870afd6c49b32f4bfc5 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/ac1c034039544870afd6c49b32f4bfc5 2024-11-28T06:54:54,597 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/87be97f39a8b4a60ad046eb60278474f to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/87be97f39a8b4a60ad046eb60278474f 2024-11-28T06:54:54,598 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/44a3908bedaa47baabe612d84097b4f4 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/44a3908bedaa47baabe612d84097b4f4 2024-11-28T06:54:54,599 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/d57e392536c64e7f883f191afc9e5bbb to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/d57e392536c64e7f883f191afc9e5bbb 2024-11-28T06:54:54,600 DEBUG [StoreCloser-TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/61fc73c2117a4670bcef1bb09f70d37d to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/61fc73c2117a4670bcef1bb09f70d37d 2024-11-28T06:54:54,603 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/recovered.edits/305.seqid, newMaxSeqId=305, maxSeqId=4 2024-11-28T06:54:54,604 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. 2024-11-28T06:54:54,604 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1676): Region close journal for 5f1556cdc2c5439575ba4a9f6cd46c4e: Waiting for close lock at 1732776892826Running coprocessor pre-close hooks at 1732776892826Disabling compacts and flushes for region at 1732776892826Disabling writes for close at 1732776892826Obtaining lock to block concurrent updates at 1732776892827 (+1 ms)Preparing flush snapshotting stores in 5f1556cdc2c5439575ba4a9f6cd46c4e at 1732776892827Finished memstore snapshotting TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732776892828 (+1 ms)Flushing stores of TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e. at 1732776892829 (+1 ms)Flushing 5f1556cdc2c5439575ba4a9f6cd46c4e/A: creating writer at 1732776892829Flushing mob file 5f1556cdc2c5439575ba4a9f6cd46c4e/A: appending metadata at 1732776892838 (+9 ms)Flushing mob file 5f1556cdc2c5439575ba4a9f6cd46c4e/A: closing flushed file at 1732776892838Flushing 5f1556cdc2c5439575ba4a9f6cd46c4e/A: appending metadata at 1732776893255 (+417 ms)Flushing 5f1556cdc2c5439575ba4a9f6cd46c4e/A: closing flushed file at 1732776893255Flushing 5f1556cdc2c5439575ba4a9f6cd46c4e/B: creating writer at 1732776893670 (+415 ms)Flushing 5f1556cdc2c5439575ba4a9f6cd46c4e/B: appending metadata at 1732776893675 (+5 ms)Flushing 5f1556cdc2c5439575ba4a9f6cd46c4e/B: closing flushed file at 1732776893675Flushing 5f1556cdc2c5439575ba4a9f6cd46c4e/C: creating writer at 1732776894090 (+415 ms)Flushing 5f1556cdc2c5439575ba4a9f6cd46c4e/C: appending metadata at 1732776894094 (+4 ms)Flushing 5f1556cdc2c5439575ba4a9f6cd46c4e/C: closing flushed file at 1732776894094Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@754042e6: reopening flushed file at 1732776894509 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70860a0d: reopening flushed file at 1732776894518 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c8306eb: reopening flushed file at 1732776894523 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 5f1556cdc2c5439575ba4a9f6cd46c4e in 1702ms, sequenceid=302, compaction requested=false at 1732776894528 (+5 ms)Writing region close event to WAL at 1732776894601 (+73 ms)Running coprocessor post-close hooks at 1732776894604 (+3 ms)Closed at 1732776894604 2024-11-28T06:54:54,606 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(157): Closed 5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:54,606 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=50 updating hbase:meta row=5f1556cdc2c5439575ba4a9f6cd46c4e, regionState=CLOSED 2024-11-28T06:54:54,607 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=50, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 because future has completed 2024-11-28T06:54:54,609 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-28T06:54:54,609 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; CloseRegionProcedure 5f1556cdc2c5439575ba4a9f6cd46c4e, server=1cb9cf437a36,43341,1732776825568 in 1.9360 sec 2024-11-28T06:54:54,611 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=49 2024-11-28T06:54:54,611 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=49, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f1556cdc2c5439575ba4a9f6cd46c4e, UNASSIGN in 1.9420 sec 2024-11-28T06:54:54,613 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-28T06:54:54,613 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9460 sec 2024-11-28T06:54:54,614 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732776894613"}]},"ts":"1732776894613"} 2024-11-28T06:54:54,615 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-28T06:54:54,615 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-28T06:54:54,617 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9590 sec 2024-11-28T06:54:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-28T06:54:54,816 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:54:54,816 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-28T06:54:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T06:54:54,818 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=52, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T06:54:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-28T06:54:54,819 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=52, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T06:54:54,823 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:54,825 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A, FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B, FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C, FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/recovered.edits] 2024-11-28T06:54:54,828 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/094f4a17e1754c9499457e0ba7c69e35 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/094f4a17e1754c9499457e0ba7c69e35 2024-11-28T06:54:54,829 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/5acf9360d55e4c5a8d7fc44efdbb7726 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/A/5acf9360d55e4c5a8d7fc44efdbb7726 2024-11-28T06:54:54,831 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/cacd9c99bbf04e2daa8fb25f15e585a1 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/cacd9c99bbf04e2daa8fb25f15e585a1 2024-11-28T06:54:54,832 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/ea32228358d344ab95c190257d0b2354 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/B/ea32228358d344ab95c190257d0b2354 2024-11-28T06:54:54,834 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/3462c54d5be44e94a2753da0eb7b97c8 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/3462c54d5be44e94a2753da0eb7b97c8 2024-11-28T06:54:54,836 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/526f6a35ccf4416cafc5c3bddfaa0794 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/C/526f6a35ccf4416cafc5c3bddfaa0794 2024-11-28T06:54:54,839 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/recovered.edits/305.seqid to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e/recovered.edits/305.seqid 2024-11-28T06:54:54,840 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:54,840 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-28T06:54:54,840 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-28T06:54:54,841 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-28T06:54:54,844 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411280512ef973c9c46fe97ad060bbd86a2c8_5f1556cdc2c5439575ba4a9f6cd46c4e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411280512ef973c9c46fe97ad060bbd86a2c8_5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:54,845 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411280ab2ac28729740059e64a52203b28f79_5f1556cdc2c5439575ba4a9f6cd46c4e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411280ab2ac28729740059e64a52203b28f79_5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:54,847 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112810728cc5d08946588cdb2a40f3395440_5f1556cdc2c5439575ba4a9f6cd46c4e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112810728cc5d08946588cdb2a40f3395440_5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:54,848 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112822101c82a3bb448b85cb648c5fb11a72_5f1556cdc2c5439575ba4a9f6cd46c4e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112822101c82a3bb448b85cb648c5fb11a72_5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:54,850 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411282cdcdb627901483498a36c5e4496b33d_5f1556cdc2c5439575ba4a9f6cd46c4e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411282cdcdb627901483498a36c5e4496b33d_5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:54,852 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411285b132c11ab1e4a0985ddd6de87e99706_5f1556cdc2c5439575ba4a9f6cd46c4e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411285b132c11ab1e4a0985ddd6de87e99706_5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:54,853 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112862c2ba81a1fe412da6a0259d61c2ebc1_5f1556cdc2c5439575ba4a9f6cd46c4e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112862c2ba81a1fe412da6a0259d61c2ebc1_5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:54,854 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288538eb3c7fd54833af3b4186ca2e8852_5f1556cdc2c5439575ba4a9f6cd46c4e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288538eb3c7fd54833af3b4186ca2e8852_5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:54,855 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288b2ea539b40e4204b585e4330f66b87d_5f1556cdc2c5439575ba4a9f6cd46c4e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288b2ea539b40e4204b585e4330f66b87d_5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:54,856 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128b5c6c3682b9c4f15b4948735c354af1c_5f1556cdc2c5439575ba4a9f6cd46c4e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128b5c6c3682b9c4f15b4948735c354af1c_5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:54,857 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128c5656ec4d1c6453390a20db9b115b9f3_5f1556cdc2c5439575ba4a9f6cd46c4e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128c5656ec4d1c6453390a20db9b115b9f3_5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:54,859 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d15cf7c887e94246aac48f9c7d8a5753_5f1556cdc2c5439575ba4a9f6cd46c4e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d15cf7c887e94246aac48f9c7d8a5753_5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:54,860 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d6098472cf5e4261a0bc3d4a284f18ba_5f1556cdc2c5439575ba4a9f6cd46c4e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d6098472cf5e4261a0bc3d4a284f18ba_5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:54,861 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128eb619962091f44d5a31903b4c6d3e48c_5f1556cdc2c5439575ba4a9f6cd46c4e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128eb619962091f44d5a31903b4c6d3e48c_5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:54,862 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128ecc237e99ba64fc4a27d5a8a094db0f5_5f1556cdc2c5439575ba4a9f6cd46c4e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128ecc237e99ba64fc4a27d5a8a094db0f5_5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:54,863 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128faf7efb39f3e4dca964d7a8b80081c49_5f1556cdc2c5439575ba4a9f6cd46c4e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128faf7efb39f3e4dca964d7a8b80081c49_5f1556cdc2c5439575ba4a9f6cd46c4e 2024-11-28T06:54:54,864 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-28T06:54:54,866 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=52, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T06:54:54,870 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-28T06:54:54,872 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-28T06:54:54,874 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=52, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T06:54:54,874 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-28T06:54:54,874 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732776894874"}]},"ts":"9223372036854775807"} 2024-11-28T06:54:54,876 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-28T06:54:54,877 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5f1556cdc2c5439575ba4a9f6cd46c4e, NAME => 'TestAcidGuarantees,,1732776858654.5f1556cdc2c5439575ba4a9f6cd46c4e.', STARTKEY => '', ENDKEY => ''}] 2024-11-28T06:54:54,877 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-28T06:54:54,877 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732776894877"}]},"ts":"9223372036854775807"} 2024-11-28T06:54:54,879 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-28T06:54:54,880 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=52, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T06:54:54,881 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 63 msec 2024-11-28T06:54:54,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-28T06:54:54,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-28T06:54:54,926 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:54:54,935 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 245) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-682725700_22 at /127.0.0.1:58404 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ce3257b-8be6-c2e3-213a-03473f81f2f4/cluster_b8990226-b232-1028-e659-af5834b9be30/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ce3257b-8be6-c2e3-213a-03473f81f2f4/cluster_b8990226-b232-1028-e659-af5834b9be30/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=384 (was 421), ProcessCount=11 (was 11), AvailableMemoryMB=7742 (was 8086) 2024-11-28T06:54:54,944 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=384, ProcessCount=11, AvailableMemoryMB=7741 2024-11-28T06:54:54,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T06:54:54,946 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T06:54:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=53, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-28T06:54:54,949 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T06:54:54,949 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:54,949 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 53 2024-11-28T06:54:54,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-28T06:54:54,950 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T06:54:54,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742035_1211 (size=1061) 2024-11-28T06:54:54,965 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7 2024-11-28T06:54:54,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742036_1212 (size=53) 2024-11-28T06:54:55,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-28T06:54:55,169 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-28T06:54:55,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-28T06:54:55,374 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T06:54:55,375 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 838099231078c8881b9868d56dccb896, disabling compactions & flushes 2024-11-28T06:54:55,375 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:54:55,375 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:54:55,375 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. after waiting 0 ms 2024-11-28T06:54:55,375 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:54:55,375 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:54:55,375 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 838099231078c8881b9868d56dccb896: Waiting for close lock at 1732776895374Disabling compacts and flushes for region at 1732776895374Disabling writes for close at 1732776895375 (+1 ms)Writing region close event to WAL at 1732776895375Closed at 1732776895375 2024-11-28T06:54:55,378 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T06:54:55,379 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732776895378"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732776895378"}]},"ts":"1732776895378"} 2024-11-28T06:54:55,382 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T06:54:55,384 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T06:54:55,384 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732776895384"}]},"ts":"1732776895384"} 2024-11-28T06:54:55,385 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-28T06:54:55,385 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=838099231078c8881b9868d56dccb896, ASSIGN}] 2024-11-28T06:54:55,386 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=838099231078c8881b9868d56dccb896, ASSIGN 2024-11-28T06:54:55,387 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=838099231078c8881b9868d56dccb896, ASSIGN; state=OFFLINE, location=1cb9cf437a36,43341,1732776825568; forceNewPlan=false, retain=false 2024-11-28T06:54:55,538 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=838099231078c8881b9868d56dccb896, regionState=OPENING, regionLocation=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:55,541 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=838099231078c8881b9868d56dccb896, ASSIGN because future has completed 2024-11-28T06:54:55,541 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; OpenRegionProcedure 838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568}] 2024-11-28T06:54:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-28T06:54:55,698 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:54:55,698 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} 2024-11-28T06:54:55,698 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 838099231078c8881b9868d56dccb896 2024-11-28T06:54:55,699 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T06:54:55,699 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 838099231078c8881b9868d56dccb896 2024-11-28T06:54:55,699 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 838099231078c8881b9868d56dccb896 2024-11-28T06:54:55,700 INFO [StoreOpener-838099231078c8881b9868d56dccb896-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 838099231078c8881b9868d56dccb896 2024-11-28T06:54:55,701 INFO [StoreOpener-838099231078c8881b9868d56dccb896-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T06:54:55,701 INFO [StoreOpener-838099231078c8881b9868d56dccb896-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 838099231078c8881b9868d56dccb896 columnFamilyName A 2024-11-28T06:54:55,702 DEBUG [StoreOpener-838099231078c8881b9868d56dccb896-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:55,702 INFO [StoreOpener-838099231078c8881b9868d56dccb896-1 {}] regionserver.HStore(327): Store=838099231078c8881b9868d56dccb896/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:54:55,702 INFO [StoreOpener-838099231078c8881b9868d56dccb896-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 838099231078c8881b9868d56dccb896 2024-11-28T06:54:55,703 INFO [StoreOpener-838099231078c8881b9868d56dccb896-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T06:54:55,703 INFO [StoreOpener-838099231078c8881b9868d56dccb896-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 838099231078c8881b9868d56dccb896 columnFamilyName B 2024-11-28T06:54:55,704 DEBUG [StoreOpener-838099231078c8881b9868d56dccb896-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:55,704 INFO [StoreOpener-838099231078c8881b9868d56dccb896-1 {}] regionserver.HStore(327): Store=838099231078c8881b9868d56dccb896/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:54:55,704 INFO [StoreOpener-838099231078c8881b9868d56dccb896-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 838099231078c8881b9868d56dccb896 2024-11-28T06:54:55,705 INFO [StoreOpener-838099231078c8881b9868d56dccb896-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T06:54:55,705 INFO [StoreOpener-838099231078c8881b9868d56dccb896-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 838099231078c8881b9868d56dccb896 columnFamilyName C 2024-11-28T06:54:55,705 DEBUG [StoreOpener-838099231078c8881b9868d56dccb896-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:54:55,706 INFO [StoreOpener-838099231078c8881b9868d56dccb896-1 {}] regionserver.HStore(327): Store=838099231078c8881b9868d56dccb896/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:54:55,706 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:54:55,706 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 838099231078c8881b9868d56dccb896 2024-11-28T06:54:55,708 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896 2024-11-28T06:54:55,708 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896 2024-11-28T06:54:55,710 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 838099231078c8881b9868d56dccb896 2024-11-28T06:54:55,710 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 838099231078c8881b9868d56dccb896 2024-11-28T06:54:55,711 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T06:54:55,713 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 838099231078c8881b9868d56dccb896 2024-11-28T06:54:55,716 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T06:54:55,716 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 838099231078c8881b9868d56dccb896; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66555667, jitterRate=-0.008243277668952942}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T06:54:55,716 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 838099231078c8881b9868d56dccb896 2024-11-28T06:54:55,717 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 838099231078c8881b9868d56dccb896: Running coprocessor pre-open hook at 1732776895699Writing region info on filesystem at 1732776895699Initializing all the Stores at 1732776895699Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732776895700 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732776895700Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732776895700Cleaning up temporary data from old regions at 1732776895710 (+10 ms)Running coprocessor post-open hooks at 1732776895716 (+6 ms)Region opened successfully at 1732776895717 (+1 ms) 2024-11-28T06:54:55,718 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., pid=55, masterSystemTime=1732776895695 2024-11-28T06:54:55,719 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:54:55,720 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:54:55,720 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=838099231078c8881b9868d56dccb896, regionState=OPEN, openSeqNum=2, regionLocation=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:55,721 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=54, state=RUNNABLE, hasLock=false; OpenRegionProcedure 838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 because future has completed 2024-11-28T06:54:55,723 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-28T06:54:55,723 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; OpenRegionProcedure 838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 in 181 msec 2024-11-28T06:54:55,725 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=53 2024-11-28T06:54:55,725 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=53, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=838099231078c8881b9868d56dccb896, ASSIGN in 338 msec 2024-11-28T06:54:55,725 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T06:54:55,726 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732776895725"}]},"ts":"1732776895725"} 2024-11-28T06:54:55,727 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-28T06:54:55,727 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T06:54:55,729 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 781 msec 2024-11-28T06:54:56,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-28T06:54:56,086 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:54:56,088 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@781611af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:56,088 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:54:56,088 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:54:56,089 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:54:56,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:54:56,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:54:56,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dbf0de8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:56,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:54:56,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:54:56,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:56,092 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53240, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:54:56,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d777034, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:56,093 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:54:56,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:54:56,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:54:56,095 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58766, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:54:56,096 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T06:54:56,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 1cb9cf437a36,37503,1732776824789 2024-11-28T06:54:56,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@135339d0 2024-11-28T06:54:56,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T06:54:56,099 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53250, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T06:54:56,101 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32f1f4e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:56,101 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:54:56,101 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:54:56,102 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:54:56,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:54:56,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:54:56,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@235206ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:56,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:54:56,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:54:56,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:56,104 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53264, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:54:56,105 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e1d40a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:56,106 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cf99f0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:56,106 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:54:56,106 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:54:56,107 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:54:56,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:54:56,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:54:56,108 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f94b088, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:56,108 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:54:56,108 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:54:56,108 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:56,109 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53280, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:54:56,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af67380, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:56,110 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6594bb9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:56,110 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:54:56,111 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:54:56,111 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:54:56,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:54:56,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:54:56,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77f12162, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:56,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:54:56,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:54:56,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:56,113 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53308, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:54:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65ede41c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:56,114 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61deb417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:56,114 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:54:56,114 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:54:56,115 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:54:56,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:54:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:54:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fadf560, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:54:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:54:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:56,117 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53324, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:54:56,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1efdc16f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:56,119 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f17e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:56,119 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:54:56,119 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:54:56,120 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:54:56,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:54:56,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:54:56,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50f01340, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:56,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:54:56,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:54:56,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:56,121 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53346, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:54:56,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@650d511e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:56,122 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7acbee0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:56,122 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:54:56,122 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:54:56,123 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:54:56,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:54:56,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:54:56,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143fa7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:56,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:54:56,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:54:56,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:56,124 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53366, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:54:56,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bcf4291, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:56,125 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46eaeeea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:56,125 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:54:56,125 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:54:56,126 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:54:56,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:54:56,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:54:56,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21f0f1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:56,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:54:56,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:54:56,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:56,127 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53380, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:54:56,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cbce46f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:56,128 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bee866e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:56,129 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:54:56,129 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:54:56,129 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:54:56,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:54:56,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:54:56,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@622dda88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:56,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:54:56,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:54:56,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:56,130 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53392, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:54:56,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74fef44e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:56,131 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@148ac527, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:56,131 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:54:56,132 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:54:56,132 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:54:56,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:54:56,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:54:56,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53666078, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:56,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:54:56,133 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:54:56,133 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:56,133 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53408, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:54:56,134 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c804e47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:56,134 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4199deaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:56,134 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:54:56,134 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:54:56,135 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:54:56,135 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:54:56,135 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:54:56,135 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cdf4ff9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:56,135 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:54:56,135 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:54:56,135 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:54:56,136 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53428, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:54:56,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763cfa25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:54:56,137 DEBUG [Thread-1010 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:54:56,137 DEBUG [Thread-1016 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:54:56,137 DEBUG [Thread-1027 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:54:56,137 DEBUG [Thread-1012 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:54:56,137 DEBUG [Thread-1025 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:54:56,138 DEBUG [Thread-1023 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:54:56,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:54:56,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:54:56,138 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:54:56,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:54:56,138 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:54:56,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:54:56,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:54:56,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:54:56,138 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:54:56,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:54:56,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:54:56,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:54:56,138 DEBUG [Thread-1014 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:54:56,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:54:56,139 DEBUG [Thread-1021 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:54:56,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-28T06:54:56,139 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58770, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:54:56,139 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58782, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:54:56,139 DEBUG [Thread-1008 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:54:56,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:54:56,140 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58768, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:54:56,140 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58816, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:54:56,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:54:56,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-28T06:54:56,140 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58788, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:54:56,140 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58800, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:54:56,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:54:56,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:54:56,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:54:56,141 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:54:56,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:54:56,141 DEBUG [Thread-1019 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:54:56,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2] 2024-11-28T06:54:56,141 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58828, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:54:56,142 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:54:56,142 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58834, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:54:56,142 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:54:56,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2] 2024-11-28T06:54:56,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:54:56,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:54:56,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2] 2024-11-28T06:54:56,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2] 2024-11-28T06:54:56,145 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58842, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:54:56,146 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58848, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:54:56,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2] 2024-11-28T06:54:56,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2] 2024-11-28T06:54:56,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2] 2024-11-28T06:54:56,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2] 2024-11-28T06:54:56,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2] 2024-11-28T06:54:56,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2] 2024-11-28T06:54:56,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 838099231078c8881b9868d56dccb896 2024-11-28T06:54:56,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 838099231078c8881b9868d56dccb896 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T06:54:56,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=A 2024-11-28T06:54:56,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:56,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=B 2024-11-28T06:54:56,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:56,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=C 2024-11-28T06:54:56,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:56,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/8c861031fcf5400fbe702fbfd3843aca is 50, key is test_row_0/A:col10/1732776896197/Put/seqid=0 2024-11-28T06:54:56,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742037_1213 (size=12001) 2024-11-28T06:54:56,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-28T06:54:56,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:56,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776956263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:56,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:56,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776956264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:56,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:56,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:56,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732776896261, max timestamp=1732776896267Num uniques 859; 2024-11-28T06:54:56,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732776896261, max timestamp=1732776896267Num uniques 859; 2024-11-28T06:54:56,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732776896261, max timestamp=1732776896267Num uniques 859; 2024-11-28T06:54:56,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T06:54:56,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T06:54:56,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T06:54:56,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-28T06:54:56,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:54:56,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. as already flushing 2024-11-28T06:54:56,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:54:56,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:56,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:56,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:56,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:56,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776956312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:56,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:56,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:56,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58816 deadline: 1732776956314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:56,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:56,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:56,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776956314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:56,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:56,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:56,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776956375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:56,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:56,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776956375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:56,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:56,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:56,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:56,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776956425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:56,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:56,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:56,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58816 deadline: 1732776956426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:56,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:56,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776956427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:56,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-28T06:54:56,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:54:56,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. as already flushing 2024-11-28T06:54:56,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:54:56,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:56,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:56,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:56,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-28T06:54:56,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:56,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776956585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:56,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:56,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776956585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:56,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:56,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:56,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-28T06:54:56,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:54:56,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. as already flushing 2024-11-28T06:54:56,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:54:56,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:56,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:56,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:56,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/8c861031fcf5400fbe702fbfd3843aca 2024-11-28T06:54:56,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:56,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776956636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:56,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:56,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776956636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:56,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:56,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:56,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58816 deadline: 1732776956636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:56,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:56,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:56,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/d5b1a4a1b92643228485b7289a890bae is 50, key is test_row_0/B:col10/1732776896197/Put/seqid=0 2024-11-28T06:54:56,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742038_1214 (size=12001) 2024-11-28T06:54:56,675 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/d5b1a4a1b92643228485b7289a890bae 2024-11-28T06:54:56,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/d8089e13cac54e559a2fe27bbd3726ca is 50, key is test_row_0/C:col10/1732776896197/Put/seqid=0 2024-11-28T06:54:56,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742039_1215 (size=12001) 2024-11-28T06:54:56,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/d8089e13cac54e559a2fe27bbd3726ca 2024-11-28T06:54:56,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/8c861031fcf5400fbe702fbfd3843aca as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/8c861031fcf5400fbe702fbfd3843aca 2024-11-28T06:54:56,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/8c861031fcf5400fbe702fbfd3843aca, entries=150, sequenceid=12, filesize=11.7 K 2024-11-28T06:54:56,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-28T06:54:56,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:54:56,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. as already flushing 2024-11-28T06:54:56,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:54:56,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:56,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:56,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/d5b1a4a1b92643228485b7289a890bae as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/d5b1a4a1b92643228485b7289a890bae 2024-11-28T06:54:56,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:56,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/d5b1a4a1b92643228485b7289a890bae, entries=150, sequenceid=12, filesize=11.7 K 2024-11-28T06:54:56,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/d8089e13cac54e559a2fe27bbd3726ca as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/d8089e13cac54e559a2fe27bbd3726ca 2024-11-28T06:54:56,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-28T06:54:56,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/d8089e13cac54e559a2fe27bbd3726ca, entries=150, sequenceid=12, filesize=11.7 K 2024-11-28T06:54:56,773 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 838099231078c8881b9868d56dccb896 in 575ms, sequenceid=12, compaction requested=false 2024-11-28T06:54:56,773 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-28T06:54:56,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 838099231078c8881b9868d56dccb896: 2024-11-28T06:54:56,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-28T06:54:56,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:54:56,910 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 838099231078c8881b9868d56dccb896 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-28T06:54:56,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 838099231078c8881b9868d56dccb896 2024-11-28T06:54:56,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. as already flushing 2024-11-28T06:54:56,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=A 2024-11-28T06:54:56,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-28T06:54:56,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=B 2024-11-28T06:54:56,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-28T06:54:56,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=C 2024-11-28T06:54:56,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-28T06:54:56,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/32a0b9ae2d1b4d29bbfd2d0d04743245 is 50, key is test_row_0/A:col10/1732776896312/Put/seqid=0 2024-11-28T06:54:56,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:56,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776956951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:56,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:56,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:56,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776956952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:56,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:56,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:56,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58816 deadline: 1732776956953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:56,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:56,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:56,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776956954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:56,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:56,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776956955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:56,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:56,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:56,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:56,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742040_1216 (size=12001) 2024-11-28T06:54:57,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776957065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:57,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58816 deadline: 1732776957065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:57,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:57,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:57,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:57,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:57,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:57,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776957066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:57,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:57,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:57,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:57,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:57,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776957066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:57,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:57,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776957066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:57,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:57,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:57,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:57,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:57,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-28T06:54:57,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58816 deadline: 1732776957276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:57,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:57,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:57,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:57,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776957276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:57,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:57,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776957276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:57,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:57,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:57,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:57,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:57,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:57,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:57,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776957276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776957276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:57,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:57,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:57,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:57,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:57,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:57,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:57,375 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/32a0b9ae2d1b4d29bbfd2d0d04743245 2024-11-28T06:54:57,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/b22a4363658d42e18c33a024db3707e2 is 50, key is test_row_0/B:col10/1732776896312/Put/seqid=0 2024-11-28T06:54:57,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742041_1217 (size=12001) 2024-11-28T06:54:57,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:57,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58816 deadline: 1732776957585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:57,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:57,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:57,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776957585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:57,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:57,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:57,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:57,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:57,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:57,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:57,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776957585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:57,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:57,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:57,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:57,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:57,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776957586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:57,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:57,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:57,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:57,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:57,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776957586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:57,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:57,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:57,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:57,787 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/b22a4363658d42e18c33a024db3707e2 2024-11-28T06:54:57,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/549722548eb244539f2246a4bdf634d4 is 50, key is test_row_0/C:col10/1732776896312/Put/seqid=0 2024-11-28T06:54:57,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742042_1218 (size=12001) 2024-11-28T06:54:58,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:58,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58816 deadline: 1732776958095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:58,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:58,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:58,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:58,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:58,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776958095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:58,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:58,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:58,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:58,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:58,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776958096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:58,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:58,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:58,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:58,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:58,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776958096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:58,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:58,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:58,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:58,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:58,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776958096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:58,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:58,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:58,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:58,199 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/549722548eb244539f2246a4bdf634d4 2024-11-28T06:54:58,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/32a0b9ae2d1b4d29bbfd2d0d04743245 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/32a0b9ae2d1b4d29bbfd2d0d04743245 2024-11-28T06:54:58,208 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/32a0b9ae2d1b4d29bbfd2d0d04743245, entries=150, sequenceid=51, filesize=11.7 K 2024-11-28T06:54:58,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/b22a4363658d42e18c33a024db3707e2 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/b22a4363658d42e18c33a024db3707e2 2024-11-28T06:54:58,213 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/b22a4363658d42e18c33a024db3707e2, entries=150, sequenceid=51, filesize=11.7 K 2024-11-28T06:54:58,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/549722548eb244539f2246a4bdf634d4 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/549722548eb244539f2246a4bdf634d4 2024-11-28T06:54:58,217 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/549722548eb244539f2246a4bdf634d4, entries=150, sequenceid=51, filesize=11.7 K 2024-11-28T06:54:58,218 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~425.48 KB/435696, currentSize=33.54 KB/34350 for 838099231078c8881b9868d56dccb896 in 1308ms, sequenceid=51, compaction requested=false 2024-11-28T06:54:58,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 838099231078c8881b9868d56dccb896: 2024-11-28T06:54:58,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:54:58,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-28T06:54:58,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-28T06:54:58,221 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-28T06:54:58,221 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0780 sec 2024-11-28T06:54:58,223 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 2.0840 sec 2024-11-28T06:54:58,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-28T06:54:58,286 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:54:58,286 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:54:58,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-28T06:54:58,288 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:54:58,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-28T06:54:58,299 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:54:58,299 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:54:58,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-28T06:54:58,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-28T06:54:58,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:54:58,452 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 838099231078c8881b9868d56dccb896 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-28T06:54:58,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=A 2024-11-28T06:54:58,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:58,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=B 2024-11-28T06:54:58,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:58,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=C 2024-11-28T06:54:58,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:54:58,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/ef97e882246c400a8a12109ab305a382 is 50, key is test_row_0/A:col10/1732776896950/Put/seqid=0 2024-11-28T06:54:58,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742043_1219 (size=12001) 2024-11-28T06:54:58,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-28T06:54:58,872 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/ef97e882246c400a8a12109ab305a382 2024-11-28T06:54:58,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/6036b7f188c34978920bd6c50910845f is 50, key is test_row_0/B:col10/1732776896950/Put/seqid=0 2024-11-28T06:54:58,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742044_1220 (size=12001) 2024-11-28T06:54:58,883 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/6036b7f188c34978920bd6c50910845f 2024-11-28T06:54:58,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/ed5bfc97169f4f039588b15a62980272 is 50, key is test_row_0/C:col10/1732776896950/Put/seqid=0 2024-11-28T06:54:58,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742045_1221 (size=12001) 2024-11-28T06:54:58,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-28T06:54:59,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 838099231078c8881b9868d56dccb896 2024-11-28T06:54:59,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. as already flushing 2024-11-28T06:54:59,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732776899124, max timestamp=1732776899133Num uniques 859; 2024-11-28T06:54:59,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T06:54:59,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732776899124, max timestamp=1732776899133Num uniques 859; 2024-11-28T06:54:59,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T06:54:59,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732776899124, max timestamp=1732776899133Num uniques 859; 2024-11-28T06:54:59,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T06:54:59,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732776899151, max timestamp=1732776899158Num uniques 1068; 2024-11-28T06:54:59,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732776899151, max timestamp=1732776899158Num uniques 1068; 2024-11-28T06:54:59,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-28T06:54:59,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-28T06:54:59,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732776899151, max timestamp=1732776899161Num uniques 1068; 2024-11-28T06:54:59,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-28T06:54:59,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:59,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776959191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:59,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732776899174, max timestamp=1732776899195Num uniques 1077; 2024-11-28T06:54:59,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732776899174, max timestamp=1732776899195Num uniques 1077; 2024-11-28T06:54:59,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-28T06:54:59,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-28T06:54:59,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:59,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732776899174, max timestamp=1732776899195Num uniques 1027; 2024-11-28T06:54:59,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-28T06:54:59,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732776899212, max timestamp=1732776899232Num uniques 1036; 2024-11-28T06:54:59,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-11-28T06:54:59,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732776899212, max timestamp=1732776899232Num uniques 1036; 2024-11-28T06:54:59,237 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732776899212, max timestamp=1732776899232Num uniques 1036; 2024-11-28T06:54:59,237 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-11-28T06:54:59,237 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-11-28T06:54:59,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:59,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776959258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:59,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:59,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:59,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776959260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:59,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:59,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:59,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58816 deadline: 1732776959264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:59,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:59,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776959265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:59,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:59,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:59,297 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/ed5bfc97169f4f039588b15a62980272 2024-11-28T06:54:59,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/ef97e882246c400a8a12109ab305a382 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/ef97e882246c400a8a12109ab305a382 2024-11-28T06:54:59,306 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/ef97e882246c400a8a12109ab305a382, entries=150, sequenceid=59, filesize=11.7 K 2024-11-28T06:54:59,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/6036b7f188c34978920bd6c50910845f as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/6036b7f188c34978920bd6c50910845f 2024-11-28T06:54:59,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:59,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776959305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:59,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:59,311 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/6036b7f188c34978920bd6c50910845f, entries=150, sequenceid=59, filesize=11.7 K 2024-11-28T06:54:59,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/ed5bfc97169f4f039588b15a62980272 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/ed5bfc97169f4f039588b15a62980272 2024-11-28T06:54:59,317 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/ed5bfc97169f4f039588b15a62980272, entries=150, sequenceid=59, filesize=11.7 K 2024-11-28T06:54:59,318 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for 838099231078c8881b9868d56dccb896 in 866ms, sequenceid=59, compaction requested=true 2024-11-28T06:54:59,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 838099231078c8881b9868d56dccb896: 2024-11-28T06:54:59,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:54:59,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-28T06:54:59,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-28T06:54:59,320 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-28T06:54:59,320 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0200 sec 2024-11-28T06:54:59,322 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.0350 sec 2024-11-28T06:54:59,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 838099231078c8881b9868d56dccb896 2024-11-28T06:54:59,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 838099231078c8881b9868d56dccb896 3/3 column families, dataSize=154.31 KB heapSize=433.55 KB 2024-11-28T06:54:59,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=A 2024-11-28T06:54:59,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-28T06:54:59,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=B 2024-11-28T06:54:59,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-28T06:54:59,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=C 2024-11-28T06:54:59,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-28T06:54:59,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/8900cc61521a40c9a54a596124144a4a is 50, key is test_row_0/A:col10/1732776899380/Put/seqid=0 2024-11-28T06:54:59,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:59,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58816 deadline: 1732776959383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:59,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:59,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:59,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:59,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776959383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:59,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776959383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:59,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:59,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:59,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:59,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776959388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:59,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:59,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742046_1222 (size=19321) 2024-11-28T06:54:59,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-28T06:54:59,426 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:54:59,426 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:54:59,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-28T06:54:59,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-28T06:54:59,428 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:54:59,429 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:54:59,429 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:54:59,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:59,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58816 deadline: 1732776959495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:59,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:59,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776959495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:59,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:59,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:59,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776959495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:59,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:59,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:59,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776959496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:59,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:59,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:59,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:59,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776959515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:59,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:59,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-28T06:54:59,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-28T06:54:59,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:54:59,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. as already flushing 2024-11-28T06:54:59,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:54:59,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:59,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:59,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:59,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:59,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58816 deadline: 1732776959706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:59,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:59,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:59,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776959706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:59,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:59,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:59,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776959706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:59,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:59,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:59,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776959706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:59,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:59,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-28T06:54:59,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:54:59,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. as already flushing 2024-11-28T06:54:59,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:54:59,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:59,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:59,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:59,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-28T06:54:59,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/8900cc61521a40c9a54a596124144a4a 2024-11-28T06:54:59,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/7543e2ef220e41e48e1f7026b4e706f1 is 50, key is test_row_0/B:col10/1732776899380/Put/seqid=0 2024-11-28T06:54:59,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742047_1223 (size=12151) 2024-11-28T06:54:59,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:54:59,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776959825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:54:59,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:54:59,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:54:59,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-28T06:54:59,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:54:59,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. as already flushing 2024-11-28T06:54:59,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:54:59,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:59,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:54:59,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:00,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:00,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58816 deadline: 1732776960015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:00,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:00,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:00,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776960016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:00,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:00,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:00,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:00,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:00,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:00,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:00,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776960016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:00,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:00,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:00,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:00,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:00,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776960016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:00,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:00,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:00,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:00,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-28T06:55:00,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:00,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. as already flushing 2024-11-28T06:55:00,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:00,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:00,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:00,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:00,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-28T06:55:00,127 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T06:55:00,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-28T06:55:00,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:00,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. as already flushing 2024-11-28T06:55:00,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:00,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:00,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:00,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:00,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/7543e2ef220e41e48e1f7026b4e706f1 2024-11-28T06:55:00,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/94dd081704a64cdd92cf1a046fd73011 is 50, key is test_row_0/C:col10/1732776899380/Put/seqid=0 2024-11-28T06:55:00,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742048_1224 (size=12151) 2024-11-28T06:55:00,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:00,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776960335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:00,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:00,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:00,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:00,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-28T06:55:00,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:00,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. as already flushing 2024-11-28T06:55:00,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:00,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:00,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:00,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:00,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-28T06:55:00,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:00,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. as already flushing 2024-11-28T06:55:00,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:00,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:00,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:00,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:00,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:00,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58816 deadline: 1732776960525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:00,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:00,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:00,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776960525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:00,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:00,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:00,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:00,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:00,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:00,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776960525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:00,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:00,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:00,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:00,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:00,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:00,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776960535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:00,538 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:00,538 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:00,538 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:00,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-28T06:55:00,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/94dd081704a64cdd92cf1a046fd73011 2024-11-28T06:55:00,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/8900cc61521a40c9a54a596124144a4a as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/8900cc61521a40c9a54a596124144a4a 2024-11-28T06:55:00,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/8900cc61521a40c9a54a596124144a4a, entries=300, sequenceid=150, filesize=18.9 K 2024-11-28T06:55:00,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/7543e2ef220e41e48e1f7026b4e706f1 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/7543e2ef220e41e48e1f7026b4e706f1 2024-11-28T06:55:00,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/7543e2ef220e41e48e1f7026b4e706f1, entries=150, sequenceid=150, filesize=11.9 K 2024-11-28T06:55:00,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/94dd081704a64cdd92cf1a046fd73011 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/94dd081704a64cdd92cf1a046fd73011 2024-11-28T06:55:00,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/94dd081704a64cdd92cf1a046fd73011, entries=150, sequenceid=150, filesize=11.9 K 2024-11-28T06:55:00,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~473.34 KB/484704, currentSize=13.42 KB/13740 for 838099231078c8881b9868d56dccb896 in 1253ms, sequenceid=150, compaction requested=true 2024-11-28T06:55:00,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 838099231078c8881b9868d56dccb896: 2024-11-28T06:55:00,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 838099231078c8881b9868d56dccb896:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:55:00,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:00,634 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T06:55:00,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 838099231078c8881b9868d56dccb896:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:55:00,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:00,634 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T06:55:00,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 838099231078c8881b9868d56dccb896:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:55:00,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:55:00,635 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48154 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T06:55:00,635 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): 838099231078c8881b9868d56dccb896/B is initiating minor compaction (all files) 2024-11-28T06:55:00,635 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 838099231078c8881b9868d56dccb896/B in TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:00,635 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/d5b1a4a1b92643228485b7289a890bae, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/b22a4363658d42e18c33a024db3707e2, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/6036b7f188c34978920bd6c50910845f, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/7543e2ef220e41e48e1f7026b4e706f1] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp, totalSize=47.0 K 2024-11-28T06:55:00,636 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55324 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T06:55:00,636 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting d5b1a4a1b92643228485b7289a890bae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732776896192 2024-11-28T06:55:00,636 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): 838099231078c8881b9868d56dccb896/A is initiating minor compaction (all files) 2024-11-28T06:55:00,636 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 838099231078c8881b9868d56dccb896/A in TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:00,636 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/8c861031fcf5400fbe702fbfd3843aca, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/32a0b9ae2d1b4d29bbfd2d0d04743245, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/ef97e882246c400a8a12109ab305a382, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/8900cc61521a40c9a54a596124144a4a] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp, totalSize=54.0 K 2024-11-28T06:55:00,636 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting b22a4363658d42e18c33a024db3707e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732776896312 2024-11-28T06:55:00,636 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c861031fcf5400fbe702fbfd3843aca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732776896192 2024-11-28T06:55:00,636 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 6036b7f188c34978920bd6c50910845f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732776896950 2024-11-28T06:55:00,637 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32a0b9ae2d1b4d29bbfd2d0d04743245, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732776896312 2024-11-28T06:55:00,637 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 7543e2ef220e41e48e1f7026b4e706f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732776899378 2024-11-28T06:55:00,637 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef97e882246c400a8a12109ab305a382, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732776896950 2024-11-28T06:55:00,637 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8900cc61521a40c9a54a596124144a4a, keycount=300, bloomtype=ROW, size=18.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732776899246 2024-11-28T06:55:00,647 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 838099231078c8881b9868d56dccb896#A#compaction#184 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:55:00,648 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/4516ffa0c9b2442781ae7042aae487c3 is 50, key is test_row_0/A:col10/1732776899380/Put/seqid=0 2024-11-28T06:55:00,649 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 838099231078c8881b9868d56dccb896#B#compaction#185 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:55:00,649 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/bd9ddba4701546e7b6f489ec5227f483 is 50, key is test_row_0/B:col10/1732776899380/Put/seqid=0 2024-11-28T06:55:00,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-28T06:55:00,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:00,652 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 838099231078c8881b9868d56dccb896 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-28T06:55:00,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=A 2024-11-28T06:55:00,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:00,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=B 2024-11-28T06:55:00,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:00,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=C 2024-11-28T06:55:00,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:00,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/fc75e73ff5bf43569e4e4a6f981d43ed is 50, key is test_row_1/A:col10/1732776899386/Put/seqid=0 2024-11-28T06:55:00,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742049_1225 (size=12289) 2024-11-28T06:55:00,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742050_1226 (size=12289) 2024-11-28T06:55:00,689 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/4516ffa0c9b2442781ae7042aae487c3 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/4516ffa0c9b2442781ae7042aae487c3 2024-11-28T06:55:00,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742051_1227 (size=7365) 2024-11-28T06:55:00,699 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 838099231078c8881b9868d56dccb896/A of 838099231078c8881b9868d56dccb896 into 4516ffa0c9b2442781ae7042aae487c3(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:55:00,699 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 838099231078c8881b9868d56dccb896: 2024-11-28T06:55:00,699 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., storeName=838099231078c8881b9868d56dccb896/A, priority=12, startTime=1732776900633; duration=0sec 2024-11-28T06:55:00,699 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:55:00,699 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 838099231078c8881b9868d56dccb896:A 2024-11-28T06:55:00,699 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T06:55:00,701 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48154 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T06:55:00,701 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): 838099231078c8881b9868d56dccb896/C is initiating minor compaction (all files) 2024-11-28T06:55:00,701 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 838099231078c8881b9868d56dccb896/C in TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:00,701 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/d8089e13cac54e559a2fe27bbd3726ca, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/549722548eb244539f2246a4bdf634d4, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/ed5bfc97169f4f039588b15a62980272, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/94dd081704a64cdd92cf1a046fd73011] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp, totalSize=47.0 K 2024-11-28T06:55:00,701 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8089e13cac54e559a2fe27bbd3726ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732776896192 2024-11-28T06:55:00,702 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 549722548eb244539f2246a4bdf634d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732776896312 2024-11-28T06:55:00,702 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed5bfc97169f4f039588b15a62980272, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732776896950 2024-11-28T06:55:00,703 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94dd081704a64cdd92cf1a046fd73011, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732776899378 2024-11-28T06:55:00,713 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 838099231078c8881b9868d56dccb896#C#compaction#187 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:55:00,713 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/2bc052bae6634953bf4f82ea373ce7d4 is 50, key is test_row_0/C:col10/1732776899380/Put/seqid=0 2024-11-28T06:55:00,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742052_1228 (size=12289) 2024-11-28T06:55:01,085 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/bd9ddba4701546e7b6f489ec5227f483 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/bd9ddba4701546e7b6f489ec5227f483 2024-11-28T06:55:01,090 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 838099231078c8881b9868d56dccb896/B of 838099231078c8881b9868d56dccb896 into bd9ddba4701546e7b6f489ec5227f483(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:55:01,090 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 838099231078c8881b9868d56dccb896: 2024-11-28T06:55:01,090 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., storeName=838099231078c8881b9868d56dccb896/B, priority=12, startTime=1732776900634; duration=0sec 2024-11-28T06:55:01,090 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:01,090 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 838099231078c8881b9868d56dccb896:B 2024-11-28T06:55:01,093 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/fc75e73ff5bf43569e4e4a6f981d43ed 2024-11-28T06:55:01,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/77718b57f53a4f23bc6ea2a13f8ded65 is 50, key is test_row_1/B:col10/1732776899386/Put/seqid=0 2024-11-28T06:55:01,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742053_1229 (size=7365) 2024-11-28T06:55:01,105 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/77718b57f53a4f23bc6ea2a13f8ded65 2024-11-28T06:55:01,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/3dcbbc7a7e4041329b07e178467e397c is 50, key is test_row_1/C:col10/1732776899386/Put/seqid=0 2024-11-28T06:55:01,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742054_1230 (size=7365) 2024-11-28T06:55:01,130 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/2bc052bae6634953bf4f82ea373ce7d4 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/2bc052bae6634953bf4f82ea373ce7d4 2024-11-28T06:55:01,135 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 838099231078c8881b9868d56dccb896/C of 838099231078c8881b9868d56dccb896 into 2bc052bae6634953bf4f82ea373ce7d4(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:55:01,135 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 838099231078c8881b9868d56dccb896: 2024-11-28T06:55:01,135 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., storeName=838099231078c8881b9868d56dccb896/C, priority=12, startTime=1732776900634; duration=0sec 2024-11-28T06:55:01,135 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:01,135 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 838099231078c8881b9868d56dccb896:C 2024-11-28T06:55:01,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 838099231078c8881b9868d56dccb896 2024-11-28T06:55:01,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. as already flushing 2024-11-28T06:55:01,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732776901375, max timestamp=1732776901399Num uniques 859; 2024-11-28T06:55:01,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T06:55:01,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732776901375, max timestamp=1732776901399Num uniques 859; 2024-11-28T06:55:01,402 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732776901375, max timestamp=1732776901399Num uniques 859; 2024-11-28T06:55:01,402 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T06:55:01,402 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T06:55:01,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732776901455, max timestamp=1732776901469Num uniques 1068; 2024-11-28T06:55:01,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-28T06:55:01,473 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732776901455, max timestamp=1732776901469Num uniques 1068; 2024-11-28T06:55:01,473 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-28T06:55:01,473 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732776901455, max timestamp=1732776901469Num uniques 1068; 2024-11-28T06:55:01,473 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-28T06:55:01,522 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/3dcbbc7a7e4041329b07e178467e397c 2024-11-28T06:55:01,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/fc75e73ff5bf43569e4e4a6f981d43ed as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/fc75e73ff5bf43569e4e4a6f981d43ed 2024-11-28T06:55:01,530 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/fc75e73ff5bf43569e4e4a6f981d43ed, entries=50, sequenceid=155, filesize=7.2 K 2024-11-28T06:55:01,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732776901502, max timestamp=1732776901529Num uniques 1027; 2024-11-28T06:55:01,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732776901502, max timestamp=1732776901529Num uniques 1027; 2024-11-28T06:55:01,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-28T06:55:01,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-28T06:55:01,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732776901502, max timestamp=1732776901529Num uniques 1027; 2024-11-28T06:55:01,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-28T06:55:01,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/77718b57f53a4f23bc6ea2a13f8ded65 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/77718b57f53a4f23bc6ea2a13f8ded65 2024-11-28T06:55:01,539 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/77718b57f53a4f23bc6ea2a13f8ded65, entries=50, sequenceid=155, filesize=7.2 K 2024-11-28T06:55:01,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/3dcbbc7a7e4041329b07e178467e397c as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/3dcbbc7a7e4041329b07e178467e397c 2024-11-28T06:55:01,545 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/3dcbbc7a7e4041329b07e178467e397c, entries=50, sequenceid=155, filesize=7.2 K 2024-11-28T06:55:01,546 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=67.09 KB/68700 for 838099231078c8881b9868d56dccb896 in 894ms, sequenceid=155, compaction requested=false 2024-11-28T06:55:01,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 838099231078c8881b9868d56dccb896: 2024-11-28T06:55:01,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:01,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-28T06:55:01,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 838099231078c8881b9868d56dccb896 2024-11-28T06:55:01,547 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 838099231078c8881b9868d56dccb896 3/3 column families, dataSize=73.80 KB heapSize=204 KB 2024-11-28T06:55:01,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-28T06:55:01,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=A 2024-11-28T06:55:01,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:55:01,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=B 2024-11-28T06:55:01,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:55:01,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=C 2024-11-28T06:55:01,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:55:01,549 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-28T06:55:01,549 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1180 sec 2024-11-28T06:55:01,551 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 2.1230 sec 2024-11-28T06:55:01,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/1c41fb20957a4e018d9c5d5c64a53e9f is 50, key is test_row_0/A:col10/1732776901529/Put/seqid=0 2024-11-28T06:55:01,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742055_1231 (size=16931) 2024-11-28T06:55:01,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/1c41fb20957a4e018d9c5d5c64a53e9f 2024-11-28T06:55:01,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/8ddeda720e544e1a974d37ffcc198d4a is 50, key is test_row_0/B:col10/1732776901529/Put/seqid=0 2024-11-28T06:55:01,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742056_1232 (size=12151) 2024-11-28T06:55:01,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-28T06:55:01,576 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:55:01,576 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:55:01,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:01,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-28T06:55:01,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776961570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:01,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:01,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:01,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776961572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:01,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:01,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:01,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-28T06:55:01,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:01,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:01,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:01,578 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:55:01,579 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:55:01,579 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:55:01,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:01,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776961575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:01,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:01,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:01,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:01,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:01,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58816 deadline: 1732776961577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:01,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:01,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:01,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:01,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:01,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776961577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:01,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:01,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:01,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:01,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-28T06:55:01,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:01,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776961685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:01,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:01,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:01,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776961686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:01,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:01,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:01,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:01,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:01,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58816 deadline: 1732776961686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:01,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:01,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:01,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:01,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:01,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:01,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:01,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776961690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:01,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:01,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776961690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:01,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:01,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:01,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:01,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:01,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:01,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:01,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-28T06:55:01,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:01,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. as already flushing 2024-11-28T06:55:01,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:01,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:01,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:01,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:01,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-28T06:55:01,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:01,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. as already flushing 2024-11-28T06:55:01,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:01,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:01,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:01,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:01,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-28T06:55:01,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:01,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776961896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:01,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:01,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776961896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:01,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:01,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776961896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:01,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:01,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58816 deadline: 1732776961896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:01,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:01,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776961896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:01,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:01,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:01,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:01,969 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/8ddeda720e544e1a974d37ffcc198d4a 2024-11-28T06:55:02,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/d7c05295167d402f858fdb7d08ebcddf is 50, key is test_row_0/C:col10/1732776901529/Put/seqid=0 2024-11-28T06:55:02,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742057_1233 (size=12151) 2024-11-28T06:55:02,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-28T06:55:02,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:02,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. as already flushing 2024-11-28T06:55:02,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:02,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:02,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:02,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:02,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-28T06:55:02,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:02,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. as already flushing 2024-11-28T06:55:02,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:02,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:02,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:02,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:02,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-28T06:55:02,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:02,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776962205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:02,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:02,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:02,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:02,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:02,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776962206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:02,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:02,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776962206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:02,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:02,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:02,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:02,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:02,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:02,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776962206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:02,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:02,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:02,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:02,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58816 deadline: 1732776962206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:02,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:02,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:02,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:02,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:02,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:02,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:02,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-28T06:55:02,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:02,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. as already flushing 2024-11-28T06:55:02,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:02,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:02,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:02,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:02,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/d7c05295167d402f858fdb7d08ebcddf 2024-11-28T06:55:02,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/1c41fb20957a4e018d9c5d5c64a53e9f as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/1c41fb20957a4e018d9c5d5c64a53e9f 2024-11-28T06:55:02,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/1c41fb20957a4e018d9c5d5c64a53e9f, entries=250, sequenceid=220, filesize=16.5 K 2024-11-28T06:55:02,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/8ddeda720e544e1a974d37ffcc198d4a as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/8ddeda720e544e1a974d37ffcc198d4a 2024-11-28T06:55:02,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/8ddeda720e544e1a974d37ffcc198d4a, entries=150, sequenceid=220, filesize=11.9 K 2024-11-28T06:55:02,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/d7c05295167d402f858fdb7d08ebcddf as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/d7c05295167d402f858fdb7d08ebcddf 2024-11-28T06:55:02,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/d7c05295167d402f858fdb7d08ebcddf, entries=150, sequenceid=220, filesize=11.9 K 2024-11-28T06:55:02,435 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~203.95 KB/208848, currentSize=114.05 KB/116790 for 838099231078c8881b9868d56dccb896 in 889ms, sequenceid=220, compaction requested=true 2024-11-28T06:55:02,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 838099231078c8881b9868d56dccb896: 2024-11-28T06:55:02,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 838099231078c8881b9868d56dccb896:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:55:02,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:02,435 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:55:02,435 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:55:02,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 838099231078c8881b9868d56dccb896:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:55:02,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:02,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 838099231078c8881b9868d56dccb896:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:55:02,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:55:02,436 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:55:02,436 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 31805 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:55:02,436 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): 838099231078c8881b9868d56dccb896/A is initiating minor compaction (all files) 2024-11-28T06:55:02,436 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): 838099231078c8881b9868d56dccb896/B is initiating minor compaction (all files) 2024-11-28T06:55:02,436 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 838099231078c8881b9868d56dccb896/B in TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:02,437 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/bd9ddba4701546e7b6f489ec5227f483, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/77718b57f53a4f23bc6ea2a13f8ded65, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/8ddeda720e544e1a974d37ffcc198d4a] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp, totalSize=31.1 K 2024-11-28T06:55:02,437 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 838099231078c8881b9868d56dccb896/A in TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:02,437 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/4516ffa0c9b2442781ae7042aae487c3, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/fc75e73ff5bf43569e4e4a6f981d43ed, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/1c41fb20957a4e018d9c5d5c64a53e9f] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp, totalSize=35.7 K 2024-11-28T06:55:02,437 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting bd9ddba4701546e7b6f489ec5227f483, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732776899378 2024-11-28T06:55:02,438 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 77718b57f53a4f23bc6ea2a13f8ded65, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732776899386 2024-11-28T06:55:02,438 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4516ffa0c9b2442781ae7042aae487c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732776899378 2024-11-28T06:55:02,438 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ddeda720e544e1a974d37ffcc198d4a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732776901529 2024-11-28T06:55:02,438 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc75e73ff5bf43569e4e4a6f981d43ed, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732776899386 2024-11-28T06:55:02,438 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c41fb20957a4e018d9c5d5c64a53e9f, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732776901529 2024-11-28T06:55:02,447 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 838099231078c8881b9868d56dccb896#B#compaction#193 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:55:02,447 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/9b40911721da4844ae3b227bc57f4b88 is 50, key is test_row_0/B:col10/1732776901529/Put/seqid=0 2024-11-28T06:55:02,452 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 838099231078c8881b9868d56dccb896#A#compaction#194 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:55:02,452 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/ac7ecbc6bc0c460f88a6e3e00b9c2bfe is 50, key is test_row_0/A:col10/1732776901529/Put/seqid=0 2024-11-28T06:55:02,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742058_1234 (size=12391) 2024-11-28T06:55:02,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742059_1235 (size=12391) 2024-11-28T06:55:02,466 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/ac7ecbc6bc0c460f88a6e3e00b9c2bfe as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/ac7ecbc6bc0c460f88a6e3e00b9c2bfe 2024-11-28T06:55:02,472 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 838099231078c8881b9868d56dccb896/A of 838099231078c8881b9868d56dccb896 into ac7ecbc6bc0c460f88a6e3e00b9c2bfe(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:55:02,472 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 838099231078c8881b9868d56dccb896: 2024-11-28T06:55:02,472 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., storeName=838099231078c8881b9868d56dccb896/A, priority=13, startTime=1732776902435; duration=0sec 2024-11-28T06:55:02,472 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:55:02,473 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 838099231078c8881b9868d56dccb896:A 2024-11-28T06:55:02,473 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:55:02,474 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 31805 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:55:02,474 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): 838099231078c8881b9868d56dccb896/C is initiating minor compaction (all files) 2024-11-28T06:55:02,474 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 838099231078c8881b9868d56dccb896/C in TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:02,474 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/2bc052bae6634953bf4f82ea373ce7d4, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/3dcbbc7a7e4041329b07e178467e397c, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/d7c05295167d402f858fdb7d08ebcddf] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp, totalSize=31.1 K 2024-11-28T06:55:02,475 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bc052bae6634953bf4f82ea373ce7d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732776899378 2024-11-28T06:55:02,475 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dcbbc7a7e4041329b07e178467e397c, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732776899386 2024-11-28T06:55:02,475 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7c05295167d402f858fdb7d08ebcddf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732776901529 2024-11-28T06:55:02,483 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 838099231078c8881b9868d56dccb896#C#compaction#195 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:55:02,484 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/179e74acfeab4990983274a5e1b685e3 is 50, key is test_row_0/C:col10/1732776901529/Put/seqid=0 2024-11-28T06:55:02,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-28T06:55:02,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:02,496 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 838099231078c8881b9868d56dccb896 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T06:55:02,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=A 2024-11-28T06:55:02,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:02,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=B 2024-11-28T06:55:02,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:02,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=C 2024-11-28T06:55:02,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:02,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742060_1236 (size=12391) 2024-11-28T06:55:02,511 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/179e74acfeab4990983274a5e1b685e3 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/179e74acfeab4990983274a5e1b685e3 2024-11-28T06:55:02,518 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 838099231078c8881b9868d56dccb896/C of 838099231078c8881b9868d56dccb896 into 179e74acfeab4990983274a5e1b685e3(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:55:02,518 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 838099231078c8881b9868d56dccb896: 2024-11-28T06:55:02,518 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., storeName=838099231078c8881b9868d56dccb896/C, priority=13, startTime=1732776902435; duration=0sec 2024-11-28T06:55:02,518 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:02,518 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 838099231078c8881b9868d56dccb896:C 2024-11-28T06:55:02,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/878b18c0c7d245bca0b16d155378dc33 is 50, key is test_row_0/A:col10/1732776901574/Put/seqid=0 2024-11-28T06:55:02,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742061_1237 (size=12151) 2024-11-28T06:55:02,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-28T06:55:02,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 838099231078c8881b9868d56dccb896 2024-11-28T06:55:02,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. as already flushing 2024-11-28T06:55:02,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:02,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776962726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:02,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:02,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:02,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776962726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:02,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:02,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:02,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:02,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58816 deadline: 1732776962727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:02,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:02,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:02,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:02,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:02,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:02,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:02,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:02,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776962731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:02,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:02,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776962731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:02,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:02,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:02,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:02,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:02,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:02,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:02,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:02,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776962836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:02,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:02,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:02,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:02,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:02,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58816 deadline: 1732776962836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:02,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:02,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:02,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:02,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:02,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776962845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:02,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:02,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:02,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776962845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:02,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:02,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:02,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:02,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:02,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:02,861 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/9b40911721da4844ae3b227bc57f4b88 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/9b40911721da4844ae3b227bc57f4b88 2024-11-28T06:55:02,865 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 838099231078c8881b9868d56dccb896/B of 838099231078c8881b9868d56dccb896 into 9b40911721da4844ae3b227bc57f4b88(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:55:02,865 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 838099231078c8881b9868d56dccb896: 2024-11-28T06:55:02,865 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., storeName=838099231078c8881b9868d56dccb896/B, priority=13, startTime=1732776902435; duration=0sec 2024-11-28T06:55:02,865 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:02,865 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 838099231078c8881b9868d56dccb896:B 2024-11-28T06:55:02,928 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/878b18c0c7d245bca0b16d155378dc33 2024-11-28T06:55:02,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/0b1a35e1c0dc49b79b189689abce7f30 is 50, key is test_row_0/B:col10/1732776901574/Put/seqid=0 2024-11-28T06:55:02,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742062_1238 (size=12151) 2024-11-28T06:55:03,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:03,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776963056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:03,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:03,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776963056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:03,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:03,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:03,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:03,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:03,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:03,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776963056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:03,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:03,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:03,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:03,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:03,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58816 deadline: 1732776963056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:03,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:03,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:03,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:03,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:03,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:03,342 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/0b1a35e1c0dc49b79b189689abce7f30 2024-11-28T06:55:03,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/14df284d1b55425bbc9ba58cd93d8aa3 is 50, key is test_row_0/C:col10/1732776901574/Put/seqid=0 2024-11-28T06:55:03,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742063_1239 (size=12151) 2024-11-28T06:55:03,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:03,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776963365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:03,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:03,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:03,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776963365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:03,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:03,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:03,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:03,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:03,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:03,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:03,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776963366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:03,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:03,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:03,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:03,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:03,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58816 deadline: 1732776963366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:03,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:03,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:03,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:03,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-28T06:55:03,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:03,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776963736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:03,739 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_0' in 838099231078c8881b9868d56dccb896 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:55:03,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:03,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:03,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:03,754 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/14df284d1b55425bbc9ba58cd93d8aa3 2024-11-28T06:55:03,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/878b18c0c7d245bca0b16d155378dc33 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/878b18c0c7d245bca0b16d155378dc33 2024-11-28T06:55:03,762 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/878b18c0c7d245bca0b16d155378dc33, entries=150, sequenceid=241, filesize=11.9 K 2024-11-28T06:55:03,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/0b1a35e1c0dc49b79b189689abce7f30 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/0b1a35e1c0dc49b79b189689abce7f30 2024-11-28T06:55:03,766 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/0b1a35e1c0dc49b79b189689abce7f30, entries=150, sequenceid=241, filesize=11.9 K 2024-11-28T06:55:03,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/14df284d1b55425bbc9ba58cd93d8aa3 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/14df284d1b55425bbc9ba58cd93d8aa3 2024-11-28T06:55:03,770 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/14df284d1b55425bbc9ba58cd93d8aa3, entries=150, sequenceid=241, filesize=11.9 K 2024-11-28T06:55:03,771 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 838099231078c8881b9868d56dccb896 in 1275ms, sequenceid=241, compaction requested=false 2024-11-28T06:55:03,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 838099231078c8881b9868d56dccb896: 2024-11-28T06:55:03,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:03,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-28T06:55:03,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-28T06:55:03,774 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-28T06:55:03,774 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1930 sec 2024-11-28T06:55:03,776 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 2.1990 sec 2024-11-28T06:55:03,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 838099231078c8881b9868d56dccb896 2024-11-28T06:55:03,877 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 838099231078c8881b9868d56dccb896 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T06:55:03,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=A 2024-11-28T06:55:03,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:03,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=B 2024-11-28T06:55:03,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:03,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=C 2024-11-28T06:55:03,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:03,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/10a193d429e14c1387a1055f2187dd0a is 50, key is test_row_0/A:col10/1732776902726/Put/seqid=0 2024-11-28T06:55:03,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742064_1240 (size=12201) 2024-11-28T06:55:03,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:03,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776963893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:03,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:03,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:03,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:03,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:03,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776963897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:03,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:03,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:03,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:03,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:03,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776963898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:03,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:03,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58816 deadline: 1732776963899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:03,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:03,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:03,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:03,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:03,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:03,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:04,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:04,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776964006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:04,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:04,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:04,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:04,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:04,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776964006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:04,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:04,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776964006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:04,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:04,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:04,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:04,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:04,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:04,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:04,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:04,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58816 deadline: 1732776964015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:04,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:04,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:04,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:04,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:04,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776964216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:04,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:04,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:04,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776964216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:04,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:04,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:04,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:04,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:04,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:04,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:04,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776964216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:04,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:04,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:04,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:04,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:04,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58816 deadline: 1732776964225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:04,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:04,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:04,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:04,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/10a193d429e14c1387a1055f2187dd0a 2024-11-28T06:55:04,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/0bd50b0aa08842c6b31ba360ad48adda is 50, key is test_row_0/B:col10/1732776902726/Put/seqid=0 2024-11-28T06:55:04,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742065_1241 (size=12201) 2024-11-28T06:55:04,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:04,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776964525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:04,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:04,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:04,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:04,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:04,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776964525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:04,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:04,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:04,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:04,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:04,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776964526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:04,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:04,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:04,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:04,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:04,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58816 deadline: 1732776964536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:04,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:04,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:04,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:04,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/0bd50b0aa08842c6b31ba360ad48adda 2024-11-28T06:55:04,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/e26e019866464b4f9cadfa6f1f1868a8 is 50, key is test_row_0/C:col10/1732776902726/Put/seqid=0 2024-11-28T06:55:04,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742066_1242 (size=12201) 2024-11-28T06:55:05,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:05,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776965036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:05,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:05,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:05,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:05,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:05,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776965036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:05,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:05,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:05,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:05,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:05,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776965036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:05,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:05,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:05,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:05,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:05,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58816 deadline: 1732776965045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:05,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:05,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:05,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:05,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/e26e019866464b4f9cadfa6f1f1868a8 2024-11-28T06:55:05,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/10a193d429e14c1387a1055f2187dd0a as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/10a193d429e14c1387a1055f2187dd0a 2024-11-28T06:55:05,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/10a193d429e14c1387a1055f2187dd0a, entries=150, sequenceid=257, filesize=11.9 K 2024-11-28T06:55:05,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/0bd50b0aa08842c6b31ba360ad48adda as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/0bd50b0aa08842c6b31ba360ad48adda 2024-11-28T06:55:05,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/0bd50b0aa08842c6b31ba360ad48adda, entries=150, sequenceid=257, filesize=11.9 K 2024-11-28T06:55:05,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/e26e019866464b4f9cadfa6f1f1868a8 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/e26e019866464b4f9cadfa6f1f1868a8 2024-11-28T06:55:05,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/e26e019866464b4f9cadfa6f1f1868a8, entries=150, sequenceid=257, filesize=11.9 K 2024-11-28T06:55:05,126 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 838099231078c8881b9868d56dccb896 in 1249ms, sequenceid=257, compaction requested=true 2024-11-28T06:55:05,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 838099231078c8881b9868d56dccb896: 2024-11-28T06:55:05,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 838099231078c8881b9868d56dccb896:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:55:05,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:05,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 838099231078c8881b9868d56dccb896:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:55:05,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:05,126 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:55:05,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 838099231078c8881b9868d56dccb896:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:55:05,126 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:55:05,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:55:05,127 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36743 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:55:05,127 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36743 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:55:05,127 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): 838099231078c8881b9868d56dccb896/A is initiating minor compaction (all files) 2024-11-28T06:55:05,127 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): 838099231078c8881b9868d56dccb896/B is initiating minor compaction (all files) 2024-11-28T06:55:05,127 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 838099231078c8881b9868d56dccb896/A in TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:05,127 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 838099231078c8881b9868d56dccb896/B in TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:05,127 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/9b40911721da4844ae3b227bc57f4b88, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/0b1a35e1c0dc49b79b189689abce7f30, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/0bd50b0aa08842c6b31ba360ad48adda] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp, totalSize=35.9 K 2024-11-28T06:55:05,127 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/ac7ecbc6bc0c460f88a6e3e00b9c2bfe, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/878b18c0c7d245bca0b16d155378dc33, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/10a193d429e14c1387a1055f2187dd0a] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp, totalSize=35.9 K 2024-11-28T06:55:05,128 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac7ecbc6bc0c460f88a6e3e00b9c2bfe, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732776901529 2024-11-28T06:55:05,128 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b40911721da4844ae3b227bc57f4b88, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732776901529 2024-11-28T06:55:05,128 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b1a35e1c0dc49b79b189689abce7f30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732776901566 2024-11-28T06:55:05,128 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 878b18c0c7d245bca0b16d155378dc33, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732776901566 2024-11-28T06:55:05,128 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bd50b0aa08842c6b31ba360ad48adda, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732776902726 2024-11-28T06:55:05,128 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10a193d429e14c1387a1055f2187dd0a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732776902726 2024-11-28T06:55:05,151 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 838099231078c8881b9868d56dccb896#A#compaction#202 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:55:05,151 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/68c448a4645a4c8ca10bb9feaf49fd62 is 50, key is test_row_0/A:col10/1732776902726/Put/seqid=0 2024-11-28T06:55:05,152 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 838099231078c8881b9868d56dccb896#B#compaction#203 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:55:05,152 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/86202f6e6f6341f1b4ff845d9cf3dd7c is 50, key is test_row_0/B:col10/1732776902726/Put/seqid=0 2024-11-28T06:55:05,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742067_1243 (size=12543) 2024-11-28T06:55:05,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742068_1244 (size=12543) 2024-11-28T06:55:05,168 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-28T06:55:05,168 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-28T06:55:05,175 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/68c448a4645a4c8ca10bb9feaf49fd62 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/68c448a4645a4c8ca10bb9feaf49fd62 2024-11-28T06:55:05,180 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 838099231078c8881b9868d56dccb896/A of 838099231078c8881b9868d56dccb896 into 68c448a4645a4c8ca10bb9feaf49fd62(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:55:05,180 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 838099231078c8881b9868d56dccb896: 2024-11-28T06:55:05,180 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., storeName=838099231078c8881b9868d56dccb896/A, priority=13, startTime=1732776905126; duration=0sec 2024-11-28T06:55:05,180 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:55:05,180 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 838099231078c8881b9868d56dccb896:A 2024-11-28T06:55:05,180 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:55:05,182 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36743 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:55:05,182 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): 838099231078c8881b9868d56dccb896/C is initiating minor compaction (all files) 2024-11-28T06:55:05,182 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 838099231078c8881b9868d56dccb896/C in TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:05,182 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/179e74acfeab4990983274a5e1b685e3, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/14df284d1b55425bbc9ba58cd93d8aa3, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/e26e019866464b4f9cadfa6f1f1868a8] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp, totalSize=35.9 K 2024-11-28T06:55:05,182 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 179e74acfeab4990983274a5e1b685e3, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732776901529 2024-11-28T06:55:05,183 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14df284d1b55425bbc9ba58cd93d8aa3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732776901566 2024-11-28T06:55:05,183 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting e26e019866464b4f9cadfa6f1f1868a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732776902726 2024-11-28T06:55:05,194 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 838099231078c8881b9868d56dccb896#C#compaction#204 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:55:05,195 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/9f03de2f08a044a5a2c8bd134e6b5c5c is 50, key is test_row_0/C:col10/1732776902726/Put/seqid=0 2024-11-28T06:55:05,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742069_1245 (size=12543) 2024-11-28T06:55:05,205 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/9f03de2f08a044a5a2c8bd134e6b5c5c as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/9f03de2f08a044a5a2c8bd134e6b5c5c 2024-11-28T06:55:05,210 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 838099231078c8881b9868d56dccb896/C of 838099231078c8881b9868d56dccb896 into 9f03de2f08a044a5a2c8bd134e6b5c5c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:55:05,210 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 838099231078c8881b9868d56dccb896: 2024-11-28T06:55:05,210 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., storeName=838099231078c8881b9868d56dccb896/C, priority=13, startTime=1732776905126; duration=0sec 2024-11-28T06:55:05,210 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:05,210 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 838099231078c8881b9868d56dccb896:C 2024-11-28T06:55:05,572 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/86202f6e6f6341f1b4ff845d9cf3dd7c as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/86202f6e6f6341f1b4ff845d9cf3dd7c 2024-11-28T06:55:05,576 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 838099231078c8881b9868d56dccb896/B of 838099231078c8881b9868d56dccb896 into 86202f6e6f6341f1b4ff845d9cf3dd7c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:55:05,576 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 838099231078c8881b9868d56dccb896: 2024-11-28T06:55:05,576 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., storeName=838099231078c8881b9868d56dccb896/B, priority=13, startTime=1732776905126; duration=0sec 2024-11-28T06:55:05,576 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:05,576 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 838099231078c8881b9868d56dccb896:B 2024-11-28T06:55:05,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-28T06:55:05,746 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:55:05,746 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:55:05,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-28T06:55:05,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-28T06:55:05,748 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:55:05,749 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:55:05,749 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:55:05,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 838099231078c8881b9868d56dccb896 2024-11-28T06:55:05,767 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 838099231078c8881b9868d56dccb896 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T06:55:05,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=A 2024-11-28T06:55:05,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:05,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=B 2024-11-28T06:55:05,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:05,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=C 2024-11-28T06:55:05,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:05,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/7557fd7f5264437c820b0f0819dd8c7b is 50, key is test_row_0/A:col10/1732776905766/Put/seqid=0 2024-11-28T06:55:05,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742070_1246 (size=14741) 2024-11-28T06:55:05,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/7557fd7f5264437c820b0f0819dd8c7b 2024-11-28T06:55:05,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/df57e1a47a70439bb62f24c8cac0cdbb is 50, key is test_row_0/B:col10/1732776905766/Put/seqid=0 2024-11-28T06:55:05,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742071_1247 (size=12301) 2024-11-28T06:55:05,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:05,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776965811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:05,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:05,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:05,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:05,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-28T06:55:05,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-28T06:55:05,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:05,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. as already flushing 2024-11-28T06:55:05,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:05,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:05,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:05,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:05,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776965936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:05,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:05,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:05,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:06,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:06,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776966046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776966045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:06,049 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_2' in 838099231078c8881b9868d56dccb896 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:55:06,049 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_1' in 838099231078c8881b9868d56dccb896 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:55:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:06,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-28T06:55:06,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:06,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. as already flushing 2024-11-28T06:55:06,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:06,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:06,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:06,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776966056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:06,059 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_0' in 838099231078c8881b9868d56dccb896 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:55:06,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:06,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:06,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-28T06:55:06,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58816 deadline: 1732776966065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:06,067 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_1' in 838099231078c8881b9868d56dccb896 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:55:06,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:06,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:06,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:06,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776966146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:06,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:06,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:06,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:06,197 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/df57e1a47a70439bb62f24c8cac0cdbb 2024-11-28T06:55:06,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-28T06:55:06,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:06,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. as already flushing 2024-11-28T06:55:06,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:06,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/97d124f207604e19b78607186ef009d1 is 50, key is test_row_0/C:col10/1732776905766/Put/seqid=0 2024-11-28T06:55:06,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:06,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:06,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742072_1248 (size=12301) 2024-11-28T06:55:06,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-28T06:55:06,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:06,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. as already flushing 2024-11-28T06:55:06,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:06,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:06,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-28T06:55:06,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:06,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776966455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:06,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:06,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:06,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:06,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-28T06:55:06,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:06,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. as already flushing 2024-11-28T06:55:06,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:06,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:06,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:06,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:06,612 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/97d124f207604e19b78607186ef009d1 2024-11-28T06:55:06,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/7557fd7f5264437c820b0f0819dd8c7b as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/7557fd7f5264437c820b0f0819dd8c7b 2024-11-28T06:55:06,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/7557fd7f5264437c820b0f0819dd8c7b, entries=200, sequenceid=279, filesize=14.4 K 2024-11-28T06:55:06,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/df57e1a47a70439bb62f24c8cac0cdbb as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/df57e1a47a70439bb62f24c8cac0cdbb 2024-11-28T06:55:06,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/df57e1a47a70439bb62f24c8cac0cdbb, entries=150, sequenceid=279, filesize=12.0 K 2024-11-28T06:55:06,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/97d124f207604e19b78607186ef009d1 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/97d124f207604e19b78607186ef009d1 2024-11-28T06:55:06,631 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/97d124f207604e19b78607186ef009d1, entries=150, sequenceid=279, filesize=12.0 K 2024-11-28T06:55:06,632 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 838099231078c8881b9868d56dccb896 in 865ms, sequenceid=279, compaction requested=false 2024-11-28T06:55:06,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 838099231078c8881b9868d56dccb896: 2024-11-28T06:55:06,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-28T06:55:06,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:06,669 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 838099231078c8881b9868d56dccb896 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T06:55:06,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=A 2024-11-28T06:55:06,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:06,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=B 2024-11-28T06:55:06,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:06,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=C 2024-11-28T06:55:06,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:06,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/50a4051ec71c47acadb520fd079ae96d is 50, key is test_row_0/A:col10/1732776905782/Put/seqid=0 2024-11-28T06:55:06,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742073_1249 (size=12301) 2024-11-28T06:55:06,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-28T06:55:07,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. as already flushing 2024-11-28T06:55:07,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 838099231078c8881b9868d56dccb896 2024-11-28T06:55:07,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:07,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776967072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:07,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:07,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:07,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:07,091 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/50a4051ec71c47acadb520fd079ae96d 2024-11-28T06:55:07,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/2a3731b0174c4322866b5154a32feae6 is 50, key is test_row_0/B:col10/1732776905782/Put/seqid=0 2024-11-28T06:55:07,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742074_1250 (size=12301) 2024-11-28T06:55:07,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:07,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776967185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:07,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:07,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:07,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:07,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:07,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776967395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:07,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:07,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:07,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:07,501 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/2a3731b0174c4322866b5154a32feae6 2024-11-28T06:55:07,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/d5df4d1285c94c619c13a4dd6f71870e is 50, key is test_row_0/C:col10/1732776905782/Put/seqid=0 2024-11-28T06:55:07,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742075_1251 (size=12301) 2024-11-28T06:55:07,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:07,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776967705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:07,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:07,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:07,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:07,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-28T06:55:07,914 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/d5df4d1285c94c619c13a4dd6f71870e 2024-11-28T06:55:07,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/50a4051ec71c47acadb520fd079ae96d as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/50a4051ec71c47acadb520fd079ae96d 2024-11-28T06:55:07,922 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/50a4051ec71c47acadb520fd079ae96d, entries=150, sequenceid=292, filesize=12.0 K 2024-11-28T06:55:07,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/2a3731b0174c4322866b5154a32feae6 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/2a3731b0174c4322866b5154a32feae6 2024-11-28T06:55:07,926 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/2a3731b0174c4322866b5154a32feae6, entries=150, sequenceid=292, filesize=12.0 K 2024-11-28T06:55:07,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/d5df4d1285c94c619c13a4dd6f71870e as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/d5df4d1285c94c619c13a4dd6f71870e 2024-11-28T06:55:07,931 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/d5df4d1285c94c619c13a4dd6f71870e, entries=150, sequenceid=292, filesize=12.0 K 2024-11-28T06:55:07,931 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 838099231078c8881b9868d56dccb896 in 1262ms, sequenceid=292, compaction requested=true 2024-11-28T06:55:07,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 838099231078c8881b9868d56dccb896: 2024-11-28T06:55:07,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:07,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-28T06:55:07,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-28T06:55:07,935 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-28T06:55:07,935 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1840 sec 2024-11-28T06:55:07,936 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 2.1890 sec 2024-11-28T06:55:08,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 838099231078c8881b9868d56dccb896 2024-11-28T06:55:08,069 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 838099231078c8881b9868d56dccb896 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T06:55:08,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=A 2024-11-28T06:55:08,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:55:08,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=B 2024-11-28T06:55:08,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:55:08,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=C 2024-11-28T06:55:08,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:55:08,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/ed628e99dcbd41ef8e8af2b9335ed6db is 50, key is test_row_0/A:col10/1732776907070/Put/seqid=0 2024-11-28T06:55:08,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742076_1252 (size=14741) 2024-11-28T06:55:08,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:08,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776968086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:08,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:08,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:08,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:08,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:08,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776968088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:08,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:08,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:08,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:08,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:08,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776968088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:08,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:08,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:08,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:08,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:08,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58816 deadline: 1732776968096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:08,100 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_1' in 838099231078c8881b9868d56dccb896 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:55:08,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:08,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:08,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:08,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:08,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776968196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:08,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:08,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:08,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:08,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:08,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776968196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:08,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:08,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:08,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:08,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:08,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776968196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:08,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:08,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:08,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:08,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:08,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776968215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:08,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:08,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:08,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:08,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:08,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776968406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:08,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:08,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776968406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:08,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:08,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:08,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:08,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776968406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:08,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:08,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:08,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:08,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:08,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:08,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/ed628e99dcbd41ef8e8af2b9335ed6db 2024-11-28T06:55:08,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/2b610920555849228cf39ed13988e6df is 50, key is test_row_0/B:col10/1732776907070/Put/seqid=0 2024-11-28T06:55:08,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742077_1253 (size=12301) 2024-11-28T06:55:08,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:08,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776968715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:08,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:08,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:08,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:08,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:08,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776968715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:08,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:08,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:08,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:08,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:08,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776968725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:08,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:08,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/2b610920555849228cf39ed13988e6df 2024-11-28T06:55:08,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/0d48e62b6b8442ed9addad2416a8f5b5 is 50, key is test_row_0/C:col10/1732776907070/Put/seqid=0 2024-11-28T06:55:08,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742078_1254 (size=12301) 2024-11-28T06:55:09,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:09,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776969226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:09,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:09,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776969226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:09,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:09,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:09,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776969226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:09,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:09,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:09,229 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_2' in 838099231078c8881b9868d56dccb896 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:55:09,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:09,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:09,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:09,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:09,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:09,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:09,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:09,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776969235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:09,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:09,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:09,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:09,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/0d48e62b6b8442ed9addad2416a8f5b5 2024-11-28T06:55:09,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/ed628e99dcbd41ef8e8af2b9335ed6db as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/ed628e99dcbd41ef8e8af2b9335ed6db 2024-11-28T06:55:09,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/ed628e99dcbd41ef8e8af2b9335ed6db, entries=200, sequenceid=313, filesize=14.4 K 2024-11-28T06:55:09,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/2b610920555849228cf39ed13988e6df as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/2b610920555849228cf39ed13988e6df 2024-11-28T06:55:09,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/2b610920555849228cf39ed13988e6df, entries=150, sequenceid=313, filesize=12.0 K 2024-11-28T06:55:09,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/0d48e62b6b8442ed9addad2416a8f5b5 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/0d48e62b6b8442ed9addad2416a8f5b5 2024-11-28T06:55:09,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/0d48e62b6b8442ed9addad2416a8f5b5, entries=150, sequenceid=313, filesize=12.0 K 2024-11-28T06:55:09,332 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 838099231078c8881b9868d56dccb896 in 1263ms, sequenceid=313, compaction requested=true 2024-11-28T06:55:09,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 838099231078c8881b9868d56dccb896: 2024-11-28T06:55:09,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 838099231078c8881b9868d56dccb896:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:55:09,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:09,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 838099231078c8881b9868d56dccb896:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:55:09,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:09,333 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T06:55:09,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 838099231078c8881b9868d56dccb896:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:55:09,333 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T06:55:09,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:55:09,335 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54326 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T06:55:09,335 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): 838099231078c8881b9868d56dccb896/A is initiating minor compaction (all files) 2024-11-28T06:55:09,335 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 838099231078c8881b9868d56dccb896/A in TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:09,335 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/68c448a4645a4c8ca10bb9feaf49fd62, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/7557fd7f5264437c820b0f0819dd8c7b, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/50a4051ec71c47acadb520fd079ae96d, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/ed628e99dcbd41ef8e8af2b9335ed6db] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp, totalSize=53.1 K 2024-11-28T06:55:09,335 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68c448a4645a4c8ca10bb9feaf49fd62, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732776902726 2024-11-28T06:55:09,336 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7557fd7f5264437c820b0f0819dd8c7b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732776903896 2024-11-28T06:55:09,336 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50a4051ec71c47acadb520fd079ae96d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732776905782 2024-11-28T06:55:09,336 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed628e99dcbd41ef8e8af2b9335ed6db, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732776907015 2024-11-28T06:55:09,339 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49446 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T06:55:09,339 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): 838099231078c8881b9868d56dccb896/B is initiating minor compaction (all files) 2024-11-28T06:55:09,339 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 838099231078c8881b9868d56dccb896/B in TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:09,339 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/86202f6e6f6341f1b4ff845d9cf3dd7c, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/df57e1a47a70439bb62f24c8cac0cdbb, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/2a3731b0174c4322866b5154a32feae6, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/2b610920555849228cf39ed13988e6df] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp, totalSize=48.3 K 2024-11-28T06:55:09,339 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 86202f6e6f6341f1b4ff845d9cf3dd7c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732776902726 2024-11-28T06:55:09,340 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting df57e1a47a70439bb62f24c8cac0cdbb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732776903896 2024-11-28T06:55:09,340 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a3731b0174c4322866b5154a32feae6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732776905782 2024-11-28T06:55:09,340 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b610920555849228cf39ed13988e6df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732776907070 2024-11-28T06:55:09,345 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 838099231078c8881b9868d56dccb896#A#compaction#214 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:55:09,345 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/9bab23f27f874531a6e94a488e393ef4 is 50, key is test_row_0/A:col10/1732776907070/Put/seqid=0 2024-11-28T06:55:09,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742079_1255 (size=12779) 2024-11-28T06:55:09,354 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 838099231078c8881b9868d56dccb896#B#compaction#215 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:55:09,354 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/abb4205a6de94b229dce4791975a5881 is 50, key is test_row_0/B:col10/1732776907070/Put/seqid=0 2024-11-28T06:55:09,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742080_1256 (size=12779) 2024-11-28T06:55:09,755 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/9bab23f27f874531a6e94a488e393ef4 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/9bab23f27f874531a6e94a488e393ef4 2024-11-28T06:55:09,759 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 838099231078c8881b9868d56dccb896/A of 838099231078c8881b9868d56dccb896 into 9bab23f27f874531a6e94a488e393ef4(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:55:09,759 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 838099231078c8881b9868d56dccb896: 2024-11-28T06:55:09,760 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., storeName=838099231078c8881b9868d56dccb896/A, priority=12, startTime=1732776909333; duration=0sec 2024-11-28T06:55:09,760 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:55:09,760 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 838099231078c8881b9868d56dccb896:A 2024-11-28T06:55:09,760 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T06:55:09,761 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49446 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T06:55:09,761 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): 838099231078c8881b9868d56dccb896/C is initiating minor compaction (all files) 2024-11-28T06:55:09,761 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 838099231078c8881b9868d56dccb896/C in TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:09,761 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/9f03de2f08a044a5a2c8bd134e6b5c5c, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/97d124f207604e19b78607186ef009d1, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/d5df4d1285c94c619c13a4dd6f71870e, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/0d48e62b6b8442ed9addad2416a8f5b5] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp, totalSize=48.3 K 2024-11-28T06:55:09,761 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f03de2f08a044a5a2c8bd134e6b5c5c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732776902726 2024-11-28T06:55:09,762 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97d124f207604e19b78607186ef009d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732776903896 2024-11-28T06:55:09,762 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5df4d1285c94c619c13a4dd6f71870e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732776905782 2024-11-28T06:55:09,762 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d48e62b6b8442ed9addad2416a8f5b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732776907070 2024-11-28T06:55:09,769 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/abb4205a6de94b229dce4791975a5881 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/abb4205a6de94b229dce4791975a5881 2024-11-28T06:55:09,772 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 838099231078c8881b9868d56dccb896#C#compaction#216 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:55:09,773 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/b983b1ad52274b3fbc29a22f616604df is 50, key is test_row_0/C:col10/1732776907070/Put/seqid=0 2024-11-28T06:55:09,773 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 838099231078c8881b9868d56dccb896/B of 838099231078c8881b9868d56dccb896 into abb4205a6de94b229dce4791975a5881(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:55:09,773 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 838099231078c8881b9868d56dccb896: 2024-11-28T06:55:09,773 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., storeName=838099231078c8881b9868d56dccb896/B, priority=12, startTime=1732776909333; duration=0sec 2024-11-28T06:55:09,773 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:09,773 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 838099231078c8881b9868d56dccb896:B 2024-11-28T06:55:09,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742081_1257 (size=12779) 2024-11-28T06:55:09,782 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/b983b1ad52274b3fbc29a22f616604df as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/b983b1ad52274b3fbc29a22f616604df 2024-11-28T06:55:09,786 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 838099231078c8881b9868d56dccb896/C of 838099231078c8881b9868d56dccb896 into b983b1ad52274b3fbc29a22f616604df(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:55:09,786 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 838099231078c8881b9868d56dccb896: 2024-11-28T06:55:09,786 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., storeName=838099231078c8881b9868d56dccb896/C, priority=12, startTime=1732776909333; duration=0sec 2024-11-28T06:55:09,786 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:09,787 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 838099231078c8881b9868d56dccb896:C 2024-11-28T06:55:09,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-28T06:55:09,916 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:55:09,916 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:55:09,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-28T06:55:09,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-28T06:55:09,918 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:55:09,919 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:55:09,919 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:55:10,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-28T06:55:10,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-28T06:55:10,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:10,071 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 838099231078c8881b9868d56dccb896 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T06:55:10,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=A 2024-11-28T06:55:10,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:10,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=B 2024-11-28T06:55:10,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:10,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=C 2024-11-28T06:55:10,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:10,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/63c83f1de5a644869600614334d5b23d is 50, key is test_row_0/A:col10/1732776908086/Put/seqid=0 2024-11-28T06:55:10,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742082_1258 (size=12301) 2024-11-28T06:55:10,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-28T06:55:10,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. as already flushing 2024-11-28T06:55:10,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 838099231078c8881b9868d56dccb896 2024-11-28T06:55:10,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:10,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732776910260, max timestamp=1732776910272Num uniques 859; 2024-11-28T06:55:10,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776970270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:10,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T06:55:10,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:10,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:10,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:10,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:10,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776970284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:10,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:10,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:10,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:10,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:10,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776970289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:10,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:10,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:10,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:10,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:10,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776970385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:10,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:10,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:10,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:10,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:10,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776970396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:10,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:10,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:10,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:10,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:10,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776970406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:10,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:10,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:10,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:10,480 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/63c83f1de5a644869600614334d5b23d 2024-11-28T06:55:10,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/a5037836a3fc4942a956294bd821f2d0 is 50, key is test_row_0/B:col10/1732776908086/Put/seqid=0 2024-11-28T06:55:10,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742083_1259 (size=12301) 2024-11-28T06:55:10,495 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/a5037836a3fc4942a956294bd821f2d0 2024-11-28T06:55:10,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/3681e36760494474801dcaeae4dd422c is 50, key is test_row_0/C:col10/1732776908086/Put/seqid=0 2024-11-28T06:55:10,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742084_1260 (size=12301) 2024-11-28T06:55:10,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-28T06:55:10,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:10,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776970595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:10,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:10,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776970606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:10,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:10,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:10,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:10,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:10,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776970615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:10,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:10,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:10,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:10,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:10,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776970905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:10,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:10,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:10,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:10,909 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/3681e36760494474801dcaeae4dd422c 2024-11-28T06:55:10,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/63c83f1de5a644869600614334d5b23d as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/63c83f1de5a644869600614334d5b23d 2024-11-28T06:55:10,918 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/63c83f1de5a644869600614334d5b23d, entries=150, sequenceid=327, filesize=12.0 K 2024-11-28T06:55:10,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/a5037836a3fc4942a956294bd821f2d0 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/a5037836a3fc4942a956294bd821f2d0 2024-11-28T06:55:10,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:10,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776970915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:10,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:10,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:10,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:10,923 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/a5037836a3fc4942a956294bd821f2d0, entries=150, sequenceid=327, filesize=12.0 K 2024-11-28T06:55:10,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/3681e36760494474801dcaeae4dd422c as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/3681e36760494474801dcaeae4dd422c 2024-11-28T06:55:10,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:10,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776970926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:10,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:10,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:10,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:10,931 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/3681e36760494474801dcaeae4dd422c, entries=150, sequenceid=327, filesize=12.0 K 2024-11-28T06:55:10,932 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=125.23 KB/128240 for 838099231078c8881b9868d56dccb896 in 861ms, sequenceid=327, compaction requested=false 2024-11-28T06:55:10,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 838099231078c8881b9868d56dccb896: 2024-11-28T06:55:10,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:10,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-28T06:55:10,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-28T06:55:10,936 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-28T06:55:10,937 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0150 sec 2024-11-28T06:55:10,939 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.0200 sec 2024-11-28T06:55:11,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-28T06:55:11,056 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:55:11,056 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:55:11,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-28T06:55:11,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-28T06:55:11,057 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:55:11,058 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:55:11,058 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:55:11,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-28T06:55:11,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-28T06:55:11,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:11,210 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 838099231078c8881b9868d56dccb896 3/3 column families, dataSize=125.23 KB heapSize=366.80 KB 2024-11-28T06:55:11,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=A 2024-11-28T06:55:11,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:55:11,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=B 2024-11-28T06:55:11,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:55:11,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=C 2024-11-28T06:55:11,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:55:11,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/d765d8c7840c45d888146304ec428933 is 50, key is test_row_0/A:col10/1732776910287/Put/seqid=0 2024-11-28T06:55:11,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742085_1261 (size=12301) 2024-11-28T06:55:11,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 838099231078c8881b9868d56dccb896 2024-11-28T06:55:11,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. as already flushing 2024-11-28T06:55:11,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:11,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776971291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:11,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:11,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:11,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:11,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-28T06:55:11,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:11,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776971406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:11,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:11,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:11,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:11,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:11,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776971415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:11,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:11,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:11,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:11,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:11,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776971425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:11,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:11,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:11,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:11,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:11,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776971435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:11,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:11,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:11,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:11,619 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/d765d8c7840c45d888146304ec428933 2024-11-28T06:55:11,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:11,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776971616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:11,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:11,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:11,620 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:11,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/50654bcaa0bf44c69a57ffd393224678 is 50, key is test_row_0/B:col10/1732776910287/Put/seqid=0 2024-11-28T06:55:11,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742086_1262 (size=12301) 2024-11-28T06:55:11,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-28T06:55:11,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:11,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776971925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:11,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:11,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:11,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:12,036 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/50654bcaa0bf44c69a57ffd393224678 2024-11-28T06:55:12,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/db5addced9b3446c8a68af926af7d06b is 50, key is test_row_0/C:col10/1732776910287/Put/seqid=0 2024-11-28T06:55:12,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742087_1263 (size=12301) 2024-11-28T06:55:12,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:12,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58816 deadline: 1732776972125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:12,129 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_1' in 838099231078c8881b9868d56dccb896 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:55:12,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:12,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:12,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-28T06:55:12,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:12,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776972425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:12,429 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_1' in 838099231078c8881b9868d56dccb896 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:55:12,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:12,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:12,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:12,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:12,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776972435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:12,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:12,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:12,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:12,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:12,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776972445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:12,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:12,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776972445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:12,449 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_0' in 838099231078c8881b9868d56dccb896 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:55:12,450 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_0' in 838099231078c8881b9868d56dccb896 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:55:12,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:12,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:12,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:12,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:12,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:12,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:12,482 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/db5addced9b3446c8a68af926af7d06b 2024-11-28T06:55:12,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/d765d8c7840c45d888146304ec428933 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/d765d8c7840c45d888146304ec428933 2024-11-28T06:55:12,491 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/d765d8c7840c45d888146304ec428933, entries=150, sequenceid=353, filesize=12.0 K 2024-11-28T06:55:12,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/50654bcaa0bf44c69a57ffd393224678 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/50654bcaa0bf44c69a57ffd393224678 2024-11-28T06:55:12,496 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/50654bcaa0bf44c69a57ffd393224678, entries=150, sequenceid=353, filesize=12.0 K 2024-11-28T06:55:12,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/db5addced9b3446c8a68af926af7d06b as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/db5addced9b3446c8a68af926af7d06b 2024-11-28T06:55:12,500 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/db5addced9b3446c8a68af926af7d06b, entries=150, sequenceid=353, filesize=12.0 K 2024-11-28T06:55:12,501 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~125.23 KB/128240, heapSize ~366.75 KB/375552, currentSize=53.67 KB/54960 for 838099231078c8881b9868d56dccb896 in 1291ms, sequenceid=353, compaction requested=true 2024-11-28T06:55:12,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 838099231078c8881b9868d56dccb896: 2024-11-28T06:55:12,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:12,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-28T06:55:12,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-28T06:55:12,504 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-28T06:55:12,504 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4440 sec 2024-11-28T06:55:12,505 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.4480 sec 2024-11-28T06:55:13,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-28T06:55:13,206 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:55:13,206 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:55:13,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-28T06:55:13,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-28T06:55:13,209 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:55:13,209 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:55:13,209 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:55:13,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-28T06:55:13,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-28T06:55:13,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:13,362 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 838099231078c8881b9868d56dccb896 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T06:55:13,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=A 2024-11-28T06:55:13,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:13,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=B 2024-11-28T06:55:13,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:13,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=C 2024-11-28T06:55:13,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:13,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/730e552648ae466b84902794b79ba8c9 is 50, key is test_row_0/A:col10/1732776911282/Put/seqid=0 2024-11-28T06:55:13,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742088_1264 (size=12301) 2024-11-28T06:55:13,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 838099231078c8881b9868d56dccb896 2024-11-28T06:55:13,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. as already flushing 2024-11-28T06:55:13,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-28T06:55:13,561 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732776913525, max timestamp=1732776913558Num uniques 859; 2024-11-28T06:55:13,562 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T06:55:13,562 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732776913525, max timestamp=1732776913558Num uniques 859; 2024-11-28T06:55:13,562 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732776913525, max timestamp=1732776913558Num uniques 859; 2024-11-28T06:55:13,562 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T06:55:13,562 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T06:55:13,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:13,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776973678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:13,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:13,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:13,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:13,772 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/730e552648ae466b84902794b79ba8c9 2024-11-28T06:55:13,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/dbe66d0f013045ffa7e5393b9ca23772 is 50, key is test_row_0/B:col10/1732776911282/Put/seqid=0 2024-11-28T06:55:13,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742089_1265 (size=12301) 2024-11-28T06:55:13,789 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/dbe66d0f013045ffa7e5393b9ca23772 2024-11-28T06:55:13,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/d82aa468b62147bab9fd1043c818cfb0 is 50, key is test_row_0/C:col10/1732776911282/Put/seqid=0 2024-11-28T06:55:13,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:13,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776973798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:13,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:13,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:13,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:13,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-28T06:55:13,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742090_1266 (size=12301) 2024-11-28T06:55:13,849 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/d82aa468b62147bab9fd1043c818cfb0 2024-11-28T06:55:13,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/730e552648ae466b84902794b79ba8c9 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/730e552648ae466b84902794b79ba8c9 2024-11-28T06:55:13,860 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/730e552648ae466b84902794b79ba8c9, entries=150, sequenceid=364, filesize=12.0 K 2024-11-28T06:55:13,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/dbe66d0f013045ffa7e5393b9ca23772 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/dbe66d0f013045ffa7e5393b9ca23772 2024-11-28T06:55:13,867 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/dbe66d0f013045ffa7e5393b9ca23772, entries=150, sequenceid=364, filesize=12.0 K 2024-11-28T06:55:13,868 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T06:55:13,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/d82aa468b62147bab9fd1043c818cfb0 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/d82aa468b62147bab9fd1043c818cfb0 2024-11-28T06:55:13,873 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/d82aa468b62147bab9fd1043c818cfb0, entries=150, sequenceid=364, filesize=12.0 K 2024-11-28T06:55:13,874 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 838099231078c8881b9868d56dccb896 in 512ms, sequenceid=364, compaction requested=true 2024-11-28T06:55:13,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 838099231078c8881b9868d56dccb896: 2024-11-28T06:55:13,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:13,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-28T06:55:13,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-28T06:55:13,881 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-28T06:55:13,881 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 669 msec 2024-11-28T06:55:13,883 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 675 msec 2024-11-28T06:55:14,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 838099231078c8881b9868d56dccb896 2024-11-28T06:55:14,011 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 838099231078c8881b9868d56dccb896 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-28T06:55:14,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=A 2024-11-28T06:55:14,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:55:14,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=B 2024-11-28T06:55:14,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:55:14,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=C 2024-11-28T06:55:14,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:55:14,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/c06c666d777d496c81e943ce39e717ee is 50, key is test_row_0/A:col10/1732776913646/Put/seqid=0 2024-11-28T06:55:14,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742091_1267 (size=14741) 2024-11-28T06:55:14,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/c06c666d777d496c81e943ce39e717ee 2024-11-28T06:55:14,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/50acf93b93624e15919b770ca994c7a1 is 50, key is test_row_0/B:col10/1732776913646/Put/seqid=0 2024-11-28T06:55:14,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742092_1268 (size=12301) 2024-11-28T06:55:14,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:14,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776974065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:14,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:14,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:14,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:14,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:14,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776974175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:14,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:14,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:14,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:14,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-28T06:55:14,346 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:55:14,346 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:55:14,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-28T06:55:14,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-28T06:55:14,348 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:55:14,349 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:55:14,349 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:55:14,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:14,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776974387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:14,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:14,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:14,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:14,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:14,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58848 deadline: 1732776974445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:14,450 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_1' in 838099231078c8881b9868d56dccb896 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:55:14,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:14,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:14,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:14,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-28T06:55:14,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/50acf93b93624e15919b770ca994c7a1 2024-11-28T06:55:14,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:14,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58782 deadline: 1732776974456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:14,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:14,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58834 deadline: 1732776974456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:14,460 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_0' in 838099231078c8881b9868d56dccb896 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:55:14,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:14,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:14,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:14,462 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_0' in 838099231078c8881b9868d56dccb896 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:55:14,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:14,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:14,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:14,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/d578d8969a704d27a3280164528c8e39 is 50, key is test_row_0/C:col10/1732776913646/Put/seqid=0 2024-11-28T06:55:14,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742093_1269 (size=12301) 2024-11-28T06:55:14,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/d578d8969a704d27a3280164528c8e39 2024-11-28T06:55:14,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/c06c666d777d496c81e943ce39e717ee as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/c06c666d777d496c81e943ce39e717ee 2024-11-28T06:55:14,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/c06c666d777d496c81e943ce39e717ee, entries=200, sequenceid=401, filesize=14.4 K 2024-11-28T06:55:14,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/50acf93b93624e15919b770ca994c7a1 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/50acf93b93624e15919b770ca994c7a1 2024-11-28T06:55:14,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-28T06:55:14,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:14,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. as already flushing 2024-11-28T06:55:14,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:14,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:14,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:14,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:14,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/50acf93b93624e15919b770ca994c7a1, entries=150, sequenceid=401, filesize=12.0 K 2024-11-28T06:55:14,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/d578d8969a704d27a3280164528c8e39 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/d578d8969a704d27a3280164528c8e39 2024-11-28T06:55:14,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/d578d8969a704d27a3280164528c8e39, entries=150, sequenceid=401, filesize=12.0 K 2024-11-28T06:55:14,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 838099231078c8881b9868d56dccb896 in 502ms, sequenceid=401, compaction requested=true 2024-11-28T06:55:14,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 838099231078c8881b9868d56dccb896: 2024-11-28T06:55:14,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 838099231078c8881b9868d56dccb896:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:55:14,513 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-28T06:55:14,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:14,513 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-28T06:55:14,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 838099231078c8881b9868d56dccb896:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:55:14,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:14,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 838099231078c8881b9868d56dccb896:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:55:14,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:55:14,514 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61983 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-28T06:55:14,514 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): 838099231078c8881b9868d56dccb896/B is initiating minor compaction (all files) 2024-11-28T06:55:14,514 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 64423 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-28T06:55:14,515 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): 838099231078c8881b9868d56dccb896/A is initiating minor compaction (all files) 2024-11-28T06:55:14,515 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 838099231078c8881b9868d56dccb896/B in TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:14,515 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 838099231078c8881b9868d56dccb896/A in TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:14,515 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/9bab23f27f874531a6e94a488e393ef4, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/63c83f1de5a644869600614334d5b23d, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/d765d8c7840c45d888146304ec428933, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/730e552648ae466b84902794b79ba8c9, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/c06c666d777d496c81e943ce39e717ee] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp, totalSize=62.9 K 2024-11-28T06:55:14,515 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/abb4205a6de94b229dce4791975a5881, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/a5037836a3fc4942a956294bd821f2d0, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/50654bcaa0bf44c69a57ffd393224678, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/dbe66d0f013045ffa7e5393b9ca23772, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/50acf93b93624e15919b770ca994c7a1] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp, totalSize=60.5 K 2024-11-28T06:55:14,515 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bab23f27f874531a6e94a488e393ef4, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732776907070 2024-11-28T06:55:14,516 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63c83f1de5a644869600614334d5b23d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732776908079 2024-11-28T06:55:14,516 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting abb4205a6de94b229dce4791975a5881, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732776907070 2024-11-28T06:55:14,517 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting d765d8c7840c45d888146304ec428933, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1732776910260 2024-11-28T06:55:14,517 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting a5037836a3fc4942a956294bd821f2d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732776908079 2024-11-28T06:55:14,517 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 730e552648ae466b84902794b79ba8c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732776911278 2024-11-28T06:55:14,518 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 50654bcaa0bf44c69a57ffd393224678, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1732776910260 2024-11-28T06:55:14,518 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting c06c666d777d496c81e943ce39e717ee, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1732776913646 2024-11-28T06:55:14,518 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting dbe66d0f013045ffa7e5393b9ca23772, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732776911278 2024-11-28T06:55:14,518 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 50acf93b93624e15919b770ca994c7a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1732776913646 2024-11-28T06:55:14,529 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 838099231078c8881b9868d56dccb896#A#compaction#229 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:55:14,529 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/c3c90c04719946f08ca81cf469ab1e82 is 50, key is test_row_0/A:col10/1732776913646/Put/seqid=0 2024-11-28T06:55:14,531 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 838099231078c8881b9868d56dccb896#B#compaction#230 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:55:14,531 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/e3d85b20871148289b39f594776a7ef5 is 50, key is test_row_0/B:col10/1732776913646/Put/seqid=0 2024-11-28T06:55:14,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742095_1271 (size=12949) 2024-11-28T06:55:14,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742094_1270 (size=12949) 2024-11-28T06:55:14,546 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/e3d85b20871148289b39f594776a7ef5 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/e3d85b20871148289b39f594776a7ef5 2024-11-28T06:55:14,548 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/c3c90c04719946f08ca81cf469ab1e82 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/c3c90c04719946f08ca81cf469ab1e82 2024-11-28T06:55:14,553 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 838099231078c8881b9868d56dccb896/B of 838099231078c8881b9868d56dccb896 into e3d85b20871148289b39f594776a7ef5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:55:14,553 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 838099231078c8881b9868d56dccb896: 2024-11-28T06:55:14,553 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., storeName=838099231078c8881b9868d56dccb896/B, priority=11, startTime=1732776914513; duration=0sec 2024-11-28T06:55:14,553 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 838099231078c8881b9868d56dccb896/A of 838099231078c8881b9868d56dccb896 into c3c90c04719946f08ca81cf469ab1e82(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:55:14,553 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 838099231078c8881b9868d56dccb896: 2024-11-28T06:55:14,553 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., storeName=838099231078c8881b9868d56dccb896/A, priority=11, startTime=1732776914513; duration=0sec 2024-11-28T06:55:14,553 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:55:14,553 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 838099231078c8881b9868d56dccb896:B 2024-11-28T06:55:14,553 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-28T06:55:14,553 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:14,553 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 838099231078c8881b9868d56dccb896:A 2024-11-28T06:55:14,554 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61983 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-28T06:55:14,554 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): 838099231078c8881b9868d56dccb896/C is initiating minor compaction (all files) 2024-11-28T06:55:14,554 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 838099231078c8881b9868d56dccb896/C in TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:14,554 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/b983b1ad52274b3fbc29a22f616604df, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/3681e36760494474801dcaeae4dd422c, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/db5addced9b3446c8a68af926af7d06b, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/d82aa468b62147bab9fd1043c818cfb0, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/d578d8969a704d27a3280164528c8e39] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp, totalSize=60.5 K 2024-11-28T06:55:14,555 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting b983b1ad52274b3fbc29a22f616604df, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732776907070 2024-11-28T06:55:14,555 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 3681e36760494474801dcaeae4dd422c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732776908079 2024-11-28T06:55:14,555 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting db5addced9b3446c8a68af926af7d06b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1732776910260 2024-11-28T06:55:14,556 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting d82aa468b62147bab9fd1043c818cfb0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732776911278 2024-11-28T06:55:14,556 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting d578d8969a704d27a3280164528c8e39, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1732776913646 2024-11-28T06:55:14,566 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 838099231078c8881b9868d56dccb896#C#compaction#231 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:55:14,566 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/2f2ab83879244c1f847df816eedcb0f0 is 50, key is test_row_0/C:col10/1732776913646/Put/seqid=0 2024-11-28T06:55:14,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742096_1272 (size=12949) 2024-11-28T06:55:14,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-28T06:55:14,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:14,657 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 838099231078c8881b9868d56dccb896 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T06:55:14,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=A 2024-11-28T06:55:14,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:14,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=B 2024-11-28T06:55:14,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:14,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=C 2024-11-28T06:55:14,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:14,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/954aa2aa2012446ca390dcb8e5abf2d4 is 50, key is test_row_0/A:col10/1732776914045/Put/seqid=0 2024-11-28T06:55:14,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-28T06:55:14,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742097_1273 (size=12301) 2024-11-28T06:55:14,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 838099231078c8881b9868d56dccb896 2024-11-28T06:55:14,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. as already flushing 2024-11-28T06:55:14,778 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732776914762, max timestamp=1732776914776Num uniques 859; 2024-11-28T06:55:14,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T06:55:14,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732776914762, max timestamp=1732776914776Num uniques 859; 2024-11-28T06:55:14,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732776914762, max timestamp=1732776914776Num uniques 859; 2024-11-28T06:55:14,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T06:55:14,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T06:55:14,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:14,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776974859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:14,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:14,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:14,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:14,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:14,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776974965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:14,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:14,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:14,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:14,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-28T06:55:14,981 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/2f2ab83879244c1f847df816eedcb0f0 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/2f2ab83879244c1f847df816eedcb0f0 2024-11-28T06:55:14,985 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 838099231078c8881b9868d56dccb896/C of 838099231078c8881b9868d56dccb896 into 2f2ab83879244c1f847df816eedcb0f0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:55:14,986 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 838099231078c8881b9868d56dccb896: 2024-11-28T06:55:14,986 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., storeName=838099231078c8881b9868d56dccb896/C, priority=11, startTime=1732776914514; duration=0sec 2024-11-28T06:55:14,986 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:14,986 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 838099231078c8881b9868d56dccb896:C 2024-11-28T06:55:15,067 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/954aa2aa2012446ca390dcb8e5abf2d4 2024-11-28T06:55:15,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/ecd7030ab362428ba1416f3b06ea4d76 is 50, key is test_row_0/B:col10/1732776914045/Put/seqid=0 2024-11-28T06:55:15,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742098_1274 (size=12301) 2024-11-28T06:55:15,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:15,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776975175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:15,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:15,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:15,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:15,479 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/ecd7030ab362428ba1416f3b06ea4d76 2024-11-28T06:55:15,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-28T06:55:15,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:15,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776975486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:15,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:15,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:15,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:15,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/19816d6935c04889b59bbb4fe4aaac02 is 50, key is test_row_0/C:col10/1732776914045/Put/seqid=0 2024-11-28T06:55:15,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742099_1275 (size=12301) 2024-11-28T06:55:15,899 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/19816d6935c04889b59bbb4fe4aaac02 2024-11-28T06:55:15,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/954aa2aa2012446ca390dcb8e5abf2d4 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/954aa2aa2012446ca390dcb8e5abf2d4 2024-11-28T06:55:15,924 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/954aa2aa2012446ca390dcb8e5abf2d4, entries=150, sequenceid=413, filesize=12.0 K 2024-11-28T06:55:15,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/ecd7030ab362428ba1416f3b06ea4d76 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/ecd7030ab362428ba1416f3b06ea4d76 2024-11-28T06:55:15,934 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/ecd7030ab362428ba1416f3b06ea4d76, entries=150, sequenceid=413, filesize=12.0 K 2024-11-28T06:55:15,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/19816d6935c04889b59bbb4fe4aaac02 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/19816d6935c04889b59bbb4fe4aaac02 2024-11-28T06:55:15,947 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/19816d6935c04889b59bbb4fe4aaac02, entries=150, sequenceid=413, filesize=12.0 K 2024-11-28T06:55:15,948 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 838099231078c8881b9868d56dccb896 in 1291ms, sequenceid=413, compaction requested=false 2024-11-28T06:55:15,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 838099231078c8881b9868d56dccb896: 2024-11-28T06:55:15,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:15,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-28T06:55:15,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-28T06:55:15,952 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-28T06:55:15,952 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6010 sec 2024-11-28T06:55:15,954 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.6070 sec 2024-11-28T06:55:15,999 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 838099231078c8881b9868d56dccb896 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-28T06:55:15,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=A 2024-11-28T06:55:15,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-28T06:55:15,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=B 2024-11-28T06:55:15,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-28T06:55:15,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=C 2024-11-28T06:55:15,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-28T06:55:15,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 838099231078c8881b9868d56dccb896 2024-11-28T06:55:16,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/3f44596d01484b0bb6e437598eea6733 is 50, key is test_row_0/A:col10/1732776914833/Put/seqid=0 2024-11-28T06:55:16,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742100_1276 (size=12301) 2024-11-28T06:55:16,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/3f44596d01484b0bb6e437598eea6733 2024-11-28T06:55:16,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/bf40d1a7c55f4d67b4b312bfcc5a5c8e is 50, key is test_row_0/B:col10/1732776914833/Put/seqid=0 2024-11-28T06:55:16,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:16,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776976045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:16,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:16,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:16,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:16,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742101_1277 (size=12301) 2024-11-28T06:55:16,140 INFO [Thread-1027 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1027. 2024-11-28T06:55:16,140 DEBUG [Thread-1027 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:55:16,140 DEBUG [Thread-1027 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:16,140 INFO [Thread-1021 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1021. 2024-11-28T06:55:16,140 DEBUG [Thread-1021 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:55:16,140 DEBUG [Thread-1021 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:16,140 DEBUG [Thread-1027 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:16,140 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:55:16,140 DEBUG [Thread-1021 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:16,141 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:55:16,141 INFO [Thread-1019 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1019. 2024-11-28T06:55:16,141 DEBUG [Thread-1019 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:55:16,141 DEBUG [Thread-1019 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:16,141 DEBUG [Thread-1019 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:16,141 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:55:16,147 INFO [Thread-1025 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1025. 2024-11-28T06:55:16,147 DEBUG [Thread-1025 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:55:16,147 INFO [Thread-1023 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1023. 2024-11-28T06:55:16,147 DEBUG [Thread-1025 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:16,147 DEBUG [Thread-1023 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:55:16,147 DEBUG [Thread-1023 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:16,148 DEBUG [Thread-1025 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:16,148 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:55:16,148 DEBUG [Thread-1023 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:16,148 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:55:16,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:16,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776976156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:16,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:16,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:16,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:16,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:16,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776976366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:16,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:16,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:16,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:16,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/bf40d1a7c55f4d67b4b312bfcc5a5c8e 2024-11-28T06:55:16,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/1fc6311f1e2a4380a45ae14212a19dc3 is 50, key is test_row_0/C:col10/1732776914833/Put/seqid=0 2024-11-28T06:55:16,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-28T06:55:16,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742102_1278 (size=12301) 2024-11-28T06:55:16,506 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:55:16,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:16,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1732776976676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:16,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:16,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:16,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:16,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/1fc6311f1e2a4380a45ae14212a19dc3 2024-11-28T06:55:16,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/3f44596d01484b0bb6e437598eea6733 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/3f44596d01484b0bb6e437598eea6733 2024-11-28T06:55:16,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/3f44596d01484b0bb6e437598eea6733, entries=150, sequenceid=452, filesize=12.0 K 2024-11-28T06:55:16,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/bf40d1a7c55f4d67b4b312bfcc5a5c8e as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/bf40d1a7c55f4d67b4b312bfcc5a5c8e 2024-11-28T06:55:16,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/bf40d1a7c55f4d67b4b312bfcc5a5c8e, entries=150, sequenceid=452, filesize=12.0 K 2024-11-28T06:55:16,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/1fc6311f1e2a4380a45ae14212a19dc3 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/1fc6311f1e2a4380a45ae14212a19dc3 2024-11-28T06:55:16,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/1fc6311f1e2a4380a45ae14212a19dc3, entries=150, sequenceid=452, filesize=12.0 K 2024-11-28T06:55:16,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 838099231078c8881b9868d56dccb896 in 932ms, sequenceid=452, compaction requested=true 2024-11-28T06:55:16,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 838099231078c8881b9868d56dccb896: 2024-11-28T06:55:16,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 838099231078c8881b9868d56dccb896:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:55:16,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:16,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 838099231078c8881b9868d56dccb896:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:55:16,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:16,932 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:55:16,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 838099231078c8881b9868d56dccb896:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:55:16,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:55:16,932 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:55:16,933 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:55:16,933 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:55:16,933 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): 838099231078c8881b9868d56dccb896/A is initiating minor compaction (all files) 2024-11-28T06:55:16,933 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): 838099231078c8881b9868d56dccb896/B is initiating minor compaction (all files) 2024-11-28T06:55:16,933 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 838099231078c8881b9868d56dccb896/A in TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:16,933 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 838099231078c8881b9868d56dccb896/B in TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:16,933 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/c3c90c04719946f08ca81cf469ab1e82, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/954aa2aa2012446ca390dcb8e5abf2d4, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/3f44596d01484b0bb6e437598eea6733] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp, totalSize=36.7 K 2024-11-28T06:55:16,933 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/e3d85b20871148289b39f594776a7ef5, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/ecd7030ab362428ba1416f3b06ea4d76, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/bf40d1a7c55f4d67b4b312bfcc5a5c8e] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp, totalSize=36.7 K 2024-11-28T06:55:16,933 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3c90c04719946f08ca81cf469ab1e82, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1732776913646 2024-11-28T06:55:16,933 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting e3d85b20871148289b39f594776a7ef5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1732776913646 2024-11-28T06:55:16,934 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 954aa2aa2012446ca390dcb8e5abf2d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732776914031 2024-11-28T06:55:16,934 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting ecd7030ab362428ba1416f3b06ea4d76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732776914031 2024-11-28T06:55:16,934 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f44596d01484b0bb6e437598eea6733, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1732776914833 2024-11-28T06:55:16,934 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting bf40d1a7c55f4d67b4b312bfcc5a5c8e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1732776914833 2024-11-28T06:55:16,941 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 838099231078c8881b9868d56dccb896#A#compaction#238 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:55:16,941 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/e0f244fb59de404fbd6cae2ee61669dc is 50, key is test_row_0/A:col10/1732776914833/Put/seqid=0 2024-11-28T06:55:16,942 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 838099231078c8881b9868d56dccb896#B#compaction#239 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:55:16,942 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/4cd5df3791d5488ebb275dae53e0afcf is 50, key is test_row_0/B:col10/1732776914833/Put/seqid=0 2024-11-28T06:55:16,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742103_1279 (size=13051) 2024-11-28T06:55:16,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742104_1280 (size=13051) 2024-11-28T06:55:17,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 838099231078c8881b9868d56dccb896 2024-11-28T06:55:17,188 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 838099231078c8881b9868d56dccb896 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T06:55:17,188 INFO [Thread-1012 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1012. 2024-11-28T06:55:17,188 DEBUG [Thread-1012 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:55:17,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=A 2024-11-28T06:55:17,188 DEBUG [Thread-1012 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:17,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:17,188 DEBUG [Thread-1012 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:17,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=B 2024-11-28T06:55:17,188 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:55:17,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:17,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=C 2024-11-28T06:55:17,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:17,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/c966ff5ba7be4f1b9a24d9ede29583e6 is 50, key is test_row_0/A:col10/1732776916039/Put/seqid=0 2024-11-28T06:55:17,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742105_1281 (size=12301) 2024-11-28T06:55:17,356 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/e0f244fb59de404fbd6cae2ee61669dc as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/e0f244fb59de404fbd6cae2ee61669dc 2024-11-28T06:55:17,357 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/4cd5df3791d5488ebb275dae53e0afcf as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/4cd5df3791d5488ebb275dae53e0afcf 2024-11-28T06:55:17,361 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 838099231078c8881b9868d56dccb896/A of 838099231078c8881b9868d56dccb896 into e0f244fb59de404fbd6cae2ee61669dc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:55:17,361 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 838099231078c8881b9868d56dccb896: 2024-11-28T06:55:17,361 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., storeName=838099231078c8881b9868d56dccb896/A, priority=13, startTime=1732776916932; duration=0sec 2024-11-28T06:55:17,361 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:55:17,361 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 838099231078c8881b9868d56dccb896:A 2024-11-28T06:55:17,361 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:55:17,362 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 838099231078c8881b9868d56dccb896/B of 838099231078c8881b9868d56dccb896 into 4cd5df3791d5488ebb275dae53e0afcf(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:55:17,362 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 838099231078c8881b9868d56dccb896: 2024-11-28T06:55:17,362 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., storeName=838099231078c8881b9868d56dccb896/B, priority=13, startTime=1732776916932; duration=0sec 2024-11-28T06:55:17,362 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:17,362 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 838099231078c8881b9868d56dccb896:B 2024-11-28T06:55:17,362 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:55:17,362 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): 838099231078c8881b9868d56dccb896/C is initiating minor compaction (all files) 2024-11-28T06:55:17,363 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 838099231078c8881b9868d56dccb896/C in TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:17,363 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/2f2ab83879244c1f847df816eedcb0f0, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/19816d6935c04889b59bbb4fe4aaac02, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/1fc6311f1e2a4380a45ae14212a19dc3] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp, totalSize=36.7 K 2024-11-28T06:55:17,363 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f2ab83879244c1f847df816eedcb0f0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1732776913646 2024-11-28T06:55:17,363 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19816d6935c04889b59bbb4fe4aaac02, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732776914031 2024-11-28T06:55:17,363 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fc6311f1e2a4380a45ae14212a19dc3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1732776914833 2024-11-28T06:55:17,371 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 838099231078c8881b9868d56dccb896#C#compaction#241 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:55:17,372 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/476632fa58424391a375f1d39220864d is 50, key is test_row_0/C:col10/1732776914833/Put/seqid=0 2024-11-28T06:55:17,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742106_1282 (size=13051) 2024-11-28T06:55:17,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/c966ff5ba7be4f1b9a24d9ede29583e6 2024-11-28T06:55:17,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/8983386121a14fb29e8cfaaed6d669cf is 50, key is test_row_0/B:col10/1732776916039/Put/seqid=0 2024-11-28T06:55:17,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742107_1283 (size=12301) 2024-11-28T06:55:17,780 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/476632fa58424391a375f1d39220864d as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/476632fa58424391a375f1d39220864d 2024-11-28T06:55:17,784 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 838099231078c8881b9868d56dccb896/C of 838099231078c8881b9868d56dccb896 into 476632fa58424391a375f1d39220864d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:55:17,784 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 838099231078c8881b9868d56dccb896: 2024-11-28T06:55:17,784 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., storeName=838099231078c8881b9868d56dccb896/C, priority=13, startTime=1732776916932; duration=0sec 2024-11-28T06:55:17,784 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:17,784 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 838099231078c8881b9868d56dccb896:C 2024-11-28T06:55:18,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/8983386121a14fb29e8cfaaed6d669cf 2024-11-28T06:55:18,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/41f655d83c974fd3a38aed7483c327c9 is 50, key is test_row_0/C:col10/1732776916039/Put/seqid=0 2024-11-28T06:55:18,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742108_1284 (size=12301) 2024-11-28T06:55:18,423 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/41f655d83c974fd3a38aed7483c327c9 2024-11-28T06:55:18,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/c966ff5ba7be4f1b9a24d9ede29583e6 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/c966ff5ba7be4f1b9a24d9ede29583e6 2024-11-28T06:55:18,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/c966ff5ba7be4f1b9a24d9ede29583e6, entries=150, sequenceid=462, filesize=12.0 K 2024-11-28T06:55:18,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/8983386121a14fb29e8cfaaed6d669cf as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/8983386121a14fb29e8cfaaed6d669cf 2024-11-28T06:55:18,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/8983386121a14fb29e8cfaaed6d669cf, entries=150, sequenceid=462, filesize=12.0 K 2024-11-28T06:55:18,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/41f655d83c974fd3a38aed7483c327c9 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/41f655d83c974fd3a38aed7483c327c9 2024-11-28T06:55:18,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/41f655d83c974fd3a38aed7483c327c9, entries=150, sequenceid=462, filesize=12.0 K 2024-11-28T06:55:18,448 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for 838099231078c8881b9868d56dccb896 in 1261ms, sequenceid=462, compaction requested=false 2024-11-28T06:55:18,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 838099231078c8881b9868d56dccb896: 2024-11-28T06:55:18,477 INFO [Thread-1014 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1014. 2024-11-28T06:55:18,477 DEBUG [Thread-1014 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:55:18,477 DEBUG [Thread-1014 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:18,477 DEBUG [Thread-1014 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:18,477 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:55:18,487 INFO [Thread-1008 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1008. 2024-11-28T06:55:18,487 DEBUG [Thread-1008 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:55:18,487 DEBUG [Thread-1008 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:18,487 DEBUG [Thread-1008 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:18,487 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:55:18,497 INFO [Thread-1016 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1016. 2024-11-28T06:55:18,497 DEBUG [Thread-1016 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:55:18,497 DEBUG [Thread-1016 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:18,497 DEBUG [Thread-1016 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:18,497 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:55:22,227 INFO [Thread-1010 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1010. 2024-11-28T06:55:22,227 DEBUG [Thread-1010 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:55:22,227 DEBUG [Thread-1010 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:22,227 DEBUG [Thread-1010 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:22,227 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:55:22,228 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-28T06:55:22,228 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 99 2024-11-28T06:55:22,228 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-28T06:55:22,228 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 147 2024-11-28T06:55:22,228 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-28T06:55:22,228 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-28T06:55:22,228 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-28T06:55:22,228 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3343 2024-11-28T06:55:22,228 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3114 2024-11-28T06:55:22,228 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3367 2024-11-28T06:55:22,228 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3350 2024-11-28T06:55:22,228 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3109 2024-11-28T06:55:22,228 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-28T06:55:22,228 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T06:55:22,228 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T06:55:22,228 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:22,228 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:22,228 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:55:22,229 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-28T06:55:22,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-28T06:55:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-28T06:55:22,232 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732776922232"}]},"ts":"1732776922232"} 2024-11-28T06:55:22,234 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-28T06:55:22,234 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-28T06:55:22,234 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T06:55:22,236 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=838099231078c8881b9868d56dccb896, UNASSIGN}] 2024-11-28T06:55:22,237 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=838099231078c8881b9868d56dccb896, UNASSIGN 2024-11-28T06:55:22,238 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=76 updating hbase:meta row=838099231078c8881b9868d56dccb896, regionState=CLOSING, regionLocation=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:22,240 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=838099231078c8881b9868d56dccb896, UNASSIGN because future has completed 2024-11-28T06:55:22,240 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T06:55:22,240 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; CloseRegionProcedure 838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568}] 2024-11-28T06:55:22,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-28T06:55:22,393 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(122): Close 838099231078c8881b9868d56dccb896 2024-11-28T06:55:22,394 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T06:55:22,394 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1722): Closing 838099231078c8881b9868d56dccb896, disabling compactions & flushes 2024-11-28T06:55:22,394 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:22,394 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:22,394 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. after waiting 0 ms 2024-11-28T06:55:22,394 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:22,394 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(2902): Flushing 838099231078c8881b9868d56dccb896 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-28T06:55:22,395 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=A 2024-11-28T06:55:22,395 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:22,395 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=B 2024-11-28T06:55:22,395 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:22,395 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 838099231078c8881b9868d56dccb896, store=C 2024-11-28T06:55:22,395 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:22,402 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/efcd493cd4374209b3a45d14a58d15a0 is 50, key is test_row_0/A:col10/1732776918496/Put/seqid=0 2024-11-28T06:55:22,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742109_1285 (size=9857) 2024-11-28T06:55:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-28T06:55:22,807 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/efcd493cd4374209b3a45d14a58d15a0 2024-11-28T06:55:22,819 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/9a9db2f9adae4a6b879adefcfd02032b is 50, key is test_row_0/B:col10/1732776918496/Put/seqid=0 2024-11-28T06:55:22,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742110_1286 (size=9857) 2024-11-28T06:55:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-28T06:55:23,223 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/9a9db2f9adae4a6b879adefcfd02032b 2024-11-28T06:55:23,233 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/5b91bf50cdb144f78bc44341fc85116d is 50, key is test_row_0/C:col10/1732776918496/Put/seqid=0 2024-11-28T06:55:23,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742111_1287 (size=9857) 2024-11-28T06:55:23,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-28T06:55:23,639 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/5b91bf50cdb144f78bc44341fc85116d 2024-11-28T06:55:23,643 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/A/efcd493cd4374209b3a45d14a58d15a0 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/efcd493cd4374209b3a45d14a58d15a0 2024-11-28T06:55:23,647 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/efcd493cd4374209b3a45d14a58d15a0, entries=100, sequenceid=472, filesize=9.6 K 2024-11-28T06:55:23,647 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/B/9a9db2f9adae4a6b879adefcfd02032b as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/9a9db2f9adae4a6b879adefcfd02032b 2024-11-28T06:55:23,651 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/9a9db2f9adae4a6b879adefcfd02032b, entries=100, sequenceid=472, filesize=9.6 K 2024-11-28T06:55:23,652 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/.tmp/C/5b91bf50cdb144f78bc44341fc85116d as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/5b91bf50cdb144f78bc44341fc85116d 2024-11-28T06:55:23,656 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/5b91bf50cdb144f78bc44341fc85116d, entries=100, sequenceid=472, filesize=9.6 K 2024-11-28T06:55:23,657 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 838099231078c8881b9868d56dccb896 in 1263ms, sequenceid=472, compaction requested=true 2024-11-28T06:55:23,657 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/8c861031fcf5400fbe702fbfd3843aca, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/32a0b9ae2d1b4d29bbfd2d0d04743245, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/ef97e882246c400a8a12109ab305a382, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/8900cc61521a40c9a54a596124144a4a, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/4516ffa0c9b2442781ae7042aae487c3, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/fc75e73ff5bf43569e4e4a6f981d43ed, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/1c41fb20957a4e018d9c5d5c64a53e9f, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/ac7ecbc6bc0c460f88a6e3e00b9c2bfe, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/878b18c0c7d245bca0b16d155378dc33, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/68c448a4645a4c8ca10bb9feaf49fd62, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/10a193d429e14c1387a1055f2187dd0a, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/7557fd7f5264437c820b0f0819dd8c7b, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/50a4051ec71c47acadb520fd079ae96d, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/ed628e99dcbd41ef8e8af2b9335ed6db, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/9bab23f27f874531a6e94a488e393ef4, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/63c83f1de5a644869600614334d5b23d, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/d765d8c7840c45d888146304ec428933, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/730e552648ae466b84902794b79ba8c9, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/c06c666d777d496c81e943ce39e717ee, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/c3c90c04719946f08ca81cf469ab1e82, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/954aa2aa2012446ca390dcb8e5abf2d4, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/3f44596d01484b0bb6e437598eea6733] to archive 2024-11-28T06:55:23,658 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T06:55:23,660 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/8c861031fcf5400fbe702fbfd3843aca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/8c861031fcf5400fbe702fbfd3843aca 2024-11-28T06:55:23,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/32a0b9ae2d1b4d29bbfd2d0d04743245 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/32a0b9ae2d1b4d29bbfd2d0d04743245 2024-11-28T06:55:23,662 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/ef97e882246c400a8a12109ab305a382 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/ef97e882246c400a8a12109ab305a382 2024-11-28T06:55:23,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/8900cc61521a40c9a54a596124144a4a to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/8900cc61521a40c9a54a596124144a4a 2024-11-28T06:55:23,664 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/4516ffa0c9b2442781ae7042aae487c3 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/4516ffa0c9b2442781ae7042aae487c3 2024-11-28T06:55:23,665 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/fc75e73ff5bf43569e4e4a6f981d43ed to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/fc75e73ff5bf43569e4e4a6f981d43ed 2024-11-28T06:55:23,666 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/1c41fb20957a4e018d9c5d5c64a53e9f to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/1c41fb20957a4e018d9c5d5c64a53e9f 2024-11-28T06:55:23,667 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/ac7ecbc6bc0c460f88a6e3e00b9c2bfe to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/ac7ecbc6bc0c460f88a6e3e00b9c2bfe 2024-11-28T06:55:23,668 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/878b18c0c7d245bca0b16d155378dc33 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/878b18c0c7d245bca0b16d155378dc33 2024-11-28T06:55:23,669 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/68c448a4645a4c8ca10bb9feaf49fd62 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/68c448a4645a4c8ca10bb9feaf49fd62 2024-11-28T06:55:23,670 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/10a193d429e14c1387a1055f2187dd0a to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/10a193d429e14c1387a1055f2187dd0a 2024-11-28T06:55:23,671 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/7557fd7f5264437c820b0f0819dd8c7b to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/7557fd7f5264437c820b0f0819dd8c7b 2024-11-28T06:55:23,673 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/50a4051ec71c47acadb520fd079ae96d to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/50a4051ec71c47acadb520fd079ae96d 2024-11-28T06:55:23,674 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/ed628e99dcbd41ef8e8af2b9335ed6db to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/ed628e99dcbd41ef8e8af2b9335ed6db 2024-11-28T06:55:23,675 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/9bab23f27f874531a6e94a488e393ef4 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/9bab23f27f874531a6e94a488e393ef4 2024-11-28T06:55:23,676 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/63c83f1de5a644869600614334d5b23d to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/63c83f1de5a644869600614334d5b23d 2024-11-28T06:55:23,677 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/d765d8c7840c45d888146304ec428933 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/d765d8c7840c45d888146304ec428933 2024-11-28T06:55:23,678 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/730e552648ae466b84902794b79ba8c9 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/730e552648ae466b84902794b79ba8c9 2024-11-28T06:55:23,679 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/c06c666d777d496c81e943ce39e717ee to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/c06c666d777d496c81e943ce39e717ee 2024-11-28T06:55:23,680 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/c3c90c04719946f08ca81cf469ab1e82 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/c3c90c04719946f08ca81cf469ab1e82 2024-11-28T06:55:23,681 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/954aa2aa2012446ca390dcb8e5abf2d4 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/954aa2aa2012446ca390dcb8e5abf2d4 2024-11-28T06:55:23,682 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/3f44596d01484b0bb6e437598eea6733 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/3f44596d01484b0bb6e437598eea6733 2024-11-28T06:55:23,684 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/d5b1a4a1b92643228485b7289a890bae, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/b22a4363658d42e18c33a024db3707e2, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/6036b7f188c34978920bd6c50910845f, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/bd9ddba4701546e7b6f489ec5227f483, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/7543e2ef220e41e48e1f7026b4e706f1, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/77718b57f53a4f23bc6ea2a13f8ded65, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/9b40911721da4844ae3b227bc57f4b88, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/8ddeda720e544e1a974d37ffcc198d4a, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/0b1a35e1c0dc49b79b189689abce7f30, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/86202f6e6f6341f1b4ff845d9cf3dd7c, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/0bd50b0aa08842c6b31ba360ad48adda, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/df57e1a47a70439bb62f24c8cac0cdbb, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/2a3731b0174c4322866b5154a32feae6, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/abb4205a6de94b229dce4791975a5881, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/2b610920555849228cf39ed13988e6df, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/a5037836a3fc4942a956294bd821f2d0, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/50654bcaa0bf44c69a57ffd393224678, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/dbe66d0f013045ffa7e5393b9ca23772, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/e3d85b20871148289b39f594776a7ef5, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/50acf93b93624e15919b770ca994c7a1, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/ecd7030ab362428ba1416f3b06ea4d76, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/bf40d1a7c55f4d67b4b312bfcc5a5c8e] to archive 2024-11-28T06:55:23,684 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T06:55:23,686 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/d5b1a4a1b92643228485b7289a890bae to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/d5b1a4a1b92643228485b7289a890bae 2024-11-28T06:55:23,687 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/b22a4363658d42e18c33a024db3707e2 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/b22a4363658d42e18c33a024db3707e2 2024-11-28T06:55:23,688 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/6036b7f188c34978920bd6c50910845f to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/6036b7f188c34978920bd6c50910845f 2024-11-28T06:55:23,689 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/bd9ddba4701546e7b6f489ec5227f483 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/bd9ddba4701546e7b6f489ec5227f483 2024-11-28T06:55:23,690 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/7543e2ef220e41e48e1f7026b4e706f1 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/7543e2ef220e41e48e1f7026b4e706f1 2024-11-28T06:55:23,691 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/77718b57f53a4f23bc6ea2a13f8ded65 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/77718b57f53a4f23bc6ea2a13f8ded65 2024-11-28T06:55:23,692 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/9b40911721da4844ae3b227bc57f4b88 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/9b40911721da4844ae3b227bc57f4b88 2024-11-28T06:55:23,693 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/8ddeda720e544e1a974d37ffcc198d4a to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/8ddeda720e544e1a974d37ffcc198d4a 2024-11-28T06:55:23,694 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/0b1a35e1c0dc49b79b189689abce7f30 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/0b1a35e1c0dc49b79b189689abce7f30 2024-11-28T06:55:23,695 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/86202f6e6f6341f1b4ff845d9cf3dd7c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/86202f6e6f6341f1b4ff845d9cf3dd7c 2024-11-28T06:55:23,696 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/0bd50b0aa08842c6b31ba360ad48adda to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/0bd50b0aa08842c6b31ba360ad48adda 2024-11-28T06:55:23,696 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/df57e1a47a70439bb62f24c8cac0cdbb to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/df57e1a47a70439bb62f24c8cac0cdbb 2024-11-28T06:55:23,697 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/2a3731b0174c4322866b5154a32feae6 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/2a3731b0174c4322866b5154a32feae6 2024-11-28T06:55:23,698 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/abb4205a6de94b229dce4791975a5881 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/abb4205a6de94b229dce4791975a5881 2024-11-28T06:55:23,699 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/2b610920555849228cf39ed13988e6df to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/2b610920555849228cf39ed13988e6df 2024-11-28T06:55:23,700 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/a5037836a3fc4942a956294bd821f2d0 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/a5037836a3fc4942a956294bd821f2d0 2024-11-28T06:55:23,700 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/50654bcaa0bf44c69a57ffd393224678 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/50654bcaa0bf44c69a57ffd393224678 2024-11-28T06:55:23,701 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/dbe66d0f013045ffa7e5393b9ca23772 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/dbe66d0f013045ffa7e5393b9ca23772 2024-11-28T06:55:23,702 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/e3d85b20871148289b39f594776a7ef5 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/e3d85b20871148289b39f594776a7ef5 2024-11-28T06:55:23,703 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/50acf93b93624e15919b770ca994c7a1 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/50acf93b93624e15919b770ca994c7a1 2024-11-28T06:55:23,704 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/ecd7030ab362428ba1416f3b06ea4d76 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/ecd7030ab362428ba1416f3b06ea4d76 2024-11-28T06:55:23,705 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/bf40d1a7c55f4d67b4b312bfcc5a5c8e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/bf40d1a7c55f4d67b4b312bfcc5a5c8e 2024-11-28T06:55:23,707 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/d8089e13cac54e559a2fe27bbd3726ca, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/549722548eb244539f2246a4bdf634d4, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/ed5bfc97169f4f039588b15a62980272, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/2bc052bae6634953bf4f82ea373ce7d4, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/94dd081704a64cdd92cf1a046fd73011, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/3dcbbc7a7e4041329b07e178467e397c, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/179e74acfeab4990983274a5e1b685e3, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/d7c05295167d402f858fdb7d08ebcddf, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/14df284d1b55425bbc9ba58cd93d8aa3, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/9f03de2f08a044a5a2c8bd134e6b5c5c, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/e26e019866464b4f9cadfa6f1f1868a8, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/97d124f207604e19b78607186ef009d1, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/d5df4d1285c94c619c13a4dd6f71870e, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/b983b1ad52274b3fbc29a22f616604df, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/0d48e62b6b8442ed9addad2416a8f5b5, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/3681e36760494474801dcaeae4dd422c, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/db5addced9b3446c8a68af926af7d06b, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/d82aa468b62147bab9fd1043c818cfb0, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/2f2ab83879244c1f847df816eedcb0f0, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/d578d8969a704d27a3280164528c8e39, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/19816d6935c04889b59bbb4fe4aaac02, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/1fc6311f1e2a4380a45ae14212a19dc3] to archive 2024-11-28T06:55:23,707 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T06:55:23,708 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/d8089e13cac54e559a2fe27bbd3726ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/d8089e13cac54e559a2fe27bbd3726ca 2024-11-28T06:55:23,710 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/549722548eb244539f2246a4bdf634d4 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/549722548eb244539f2246a4bdf634d4 2024-11-28T06:55:23,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/ed5bfc97169f4f039588b15a62980272 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/ed5bfc97169f4f039588b15a62980272 2024-11-28T06:55:23,712 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/2bc052bae6634953bf4f82ea373ce7d4 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/2bc052bae6634953bf4f82ea373ce7d4 2024-11-28T06:55:23,713 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/94dd081704a64cdd92cf1a046fd73011 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/94dd081704a64cdd92cf1a046fd73011 2024-11-28T06:55:23,714 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/3dcbbc7a7e4041329b07e178467e397c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/3dcbbc7a7e4041329b07e178467e397c 2024-11-28T06:55:23,715 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/179e74acfeab4990983274a5e1b685e3 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/179e74acfeab4990983274a5e1b685e3 2024-11-28T06:55:23,716 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/d7c05295167d402f858fdb7d08ebcddf to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/d7c05295167d402f858fdb7d08ebcddf 2024-11-28T06:55:23,717 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/14df284d1b55425bbc9ba58cd93d8aa3 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/14df284d1b55425bbc9ba58cd93d8aa3 2024-11-28T06:55:23,718 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/9f03de2f08a044a5a2c8bd134e6b5c5c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/9f03de2f08a044a5a2c8bd134e6b5c5c 2024-11-28T06:55:23,719 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/e26e019866464b4f9cadfa6f1f1868a8 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/e26e019866464b4f9cadfa6f1f1868a8 2024-11-28T06:55:23,720 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/97d124f207604e19b78607186ef009d1 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/97d124f207604e19b78607186ef009d1 2024-11-28T06:55:23,721 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/d5df4d1285c94c619c13a4dd6f71870e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/d5df4d1285c94c619c13a4dd6f71870e 2024-11-28T06:55:23,722 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/b983b1ad52274b3fbc29a22f616604df to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/b983b1ad52274b3fbc29a22f616604df 2024-11-28T06:55:23,723 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/0d48e62b6b8442ed9addad2416a8f5b5 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/0d48e62b6b8442ed9addad2416a8f5b5 2024-11-28T06:55:23,724 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/3681e36760494474801dcaeae4dd422c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/3681e36760494474801dcaeae4dd422c 2024-11-28T06:55:23,725 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/db5addced9b3446c8a68af926af7d06b to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/db5addced9b3446c8a68af926af7d06b 2024-11-28T06:55:23,725 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/d82aa468b62147bab9fd1043c818cfb0 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/d82aa468b62147bab9fd1043c818cfb0 2024-11-28T06:55:23,726 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/2f2ab83879244c1f847df816eedcb0f0 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/2f2ab83879244c1f847df816eedcb0f0 2024-11-28T06:55:23,727 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/d578d8969a704d27a3280164528c8e39 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/d578d8969a704d27a3280164528c8e39 2024-11-28T06:55:23,728 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/19816d6935c04889b59bbb4fe4aaac02 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/19816d6935c04889b59bbb4fe4aaac02 2024-11-28T06:55:23,729 DEBUG [StoreCloser-TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/1fc6311f1e2a4380a45ae14212a19dc3 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/1fc6311f1e2a4380a45ae14212a19dc3 2024-11-28T06:55:23,733 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/recovered.edits/475.seqid, newMaxSeqId=475, maxSeqId=1 2024-11-28T06:55:23,733 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. 2024-11-28T06:55:23,733 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1676): Region close journal for 838099231078c8881b9868d56dccb896: Waiting for close lock at 1732776922394Running coprocessor pre-close hooks at 1732776922394Disabling compacts and flushes for region at 1732776922394Disabling writes for close at 1732776922394Obtaining lock to block concurrent updates at 1732776922394Preparing flush snapshotting stores in 838099231078c8881b9868d56dccb896 at 1732776922394Finished memstore snapshotting TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732776922395 (+1 ms)Flushing stores of TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896. at 1732776922396 (+1 ms)Flushing 838099231078c8881b9868d56dccb896/A: creating writer at 1732776922396Flushing 838099231078c8881b9868d56dccb896/A: appending metadata at 1732776922401 (+5 ms)Flushing 838099231078c8881b9868d56dccb896/A: closing flushed file at 1732776922401Flushing 838099231078c8881b9868d56dccb896/B: creating writer at 1732776922814 (+413 ms)Flushing 838099231078c8881b9868d56dccb896/B: appending metadata at 1732776922818 (+4 ms)Flushing 838099231078c8881b9868d56dccb896/B: closing flushed file at 1732776922818Flushing 838099231078c8881b9868d56dccb896/C: creating writer at 1732776923229 (+411 ms)Flushing 838099231078c8881b9868d56dccb896/C: appending metadata at 1732776923233 (+4 ms)Flushing 838099231078c8881b9868d56dccb896/C: closing flushed file at 1732776923233Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53e4310c: reopening flushed file at 1732776923642 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@656e30f7: reopening flushed file at 1732776923647 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e6ee0ba: reopening flushed file at 1732776923651 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 838099231078c8881b9868d56dccb896 in 1263ms, sequenceid=472, compaction requested=true at 1732776923657 (+6 ms)Writing region close event to WAL at 1732776923730 (+73 ms)Running coprocessor post-close hooks at 1732776923733 (+3 ms)Closed at 1732776923733 2024-11-28T06:55:23,735 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(157): Closed 838099231078c8881b9868d56dccb896 2024-11-28T06:55:23,736 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=76 updating hbase:meta row=838099231078c8881b9868d56dccb896, regionState=CLOSED 2024-11-28T06:55:23,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=77, ppid=76, state=RUNNABLE, hasLock=false; CloseRegionProcedure 838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 because future has completed 2024-11-28T06:55:23,740 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-28T06:55:23,740 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; CloseRegionProcedure 838099231078c8881b9868d56dccb896, server=1cb9cf437a36,43341,1732776825568 in 1.4990 sec 2024-11-28T06:55:23,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=76, resume processing ppid=75 2024-11-28T06:55:23,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, ppid=75, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=838099231078c8881b9868d56dccb896, UNASSIGN in 1.5040 sec 2024-11-28T06:55:23,743 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-28T06:55:23,744 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5080 sec 2024-11-28T06:55:23,745 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732776923744"}]},"ts":"1732776923744"} 2024-11-28T06:55:23,746 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-28T06:55:23,746 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-28T06:55:23,748 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5170 sec 2024-11-28T06:55:24,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-28T06:55:24,386 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:55:24,386 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-28T06:55:24,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T06:55:24,431 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T06:55:24,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-28T06:55:24,432 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=78, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T06:55:24,435 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896 2024-11-28T06:55:24,437 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A, FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B, FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C, FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/recovered.edits] 2024-11-28T06:55:24,440 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/c966ff5ba7be4f1b9a24d9ede29583e6 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/c966ff5ba7be4f1b9a24d9ede29583e6 2024-11-28T06:55:24,441 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/e0f244fb59de404fbd6cae2ee61669dc to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/e0f244fb59de404fbd6cae2ee61669dc 2024-11-28T06:55:24,443 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/efcd493cd4374209b3a45d14a58d15a0 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/A/efcd493cd4374209b3a45d14a58d15a0 2024-11-28T06:55:24,445 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/4cd5df3791d5488ebb275dae53e0afcf to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/4cd5df3791d5488ebb275dae53e0afcf 2024-11-28T06:55:24,446 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/8983386121a14fb29e8cfaaed6d669cf to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/8983386121a14fb29e8cfaaed6d669cf 2024-11-28T06:55:24,446 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/9a9db2f9adae4a6b879adefcfd02032b to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/B/9a9db2f9adae4a6b879adefcfd02032b 2024-11-28T06:55:24,448 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/41f655d83c974fd3a38aed7483c327c9 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/41f655d83c974fd3a38aed7483c327c9 2024-11-28T06:55:24,449 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/476632fa58424391a375f1d39220864d to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/476632fa58424391a375f1d39220864d 2024-11-28T06:55:24,450 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/5b91bf50cdb144f78bc44341fc85116d to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/C/5b91bf50cdb144f78bc44341fc85116d 2024-11-28T06:55:24,452 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/recovered.edits/475.seqid to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896/recovered.edits/475.seqid 2024-11-28T06:55:24,453 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/838099231078c8881b9868d56dccb896 2024-11-28T06:55:24,453 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-28T06:55:24,455 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=78, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T06:55:24,458 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-28T06:55:24,460 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-28T06:55:24,461 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=78, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T06:55:24,461 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-28T06:55:24,461 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732776924461"}]},"ts":"9223372036854775807"} 2024-11-28T06:55:24,463 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-28T06:55:24,463 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 838099231078c8881b9868d56dccb896, NAME => 'TestAcidGuarantees,,1732776894945.838099231078c8881b9868d56dccb896.', STARTKEY => '', ENDKEY => ''}] 2024-11-28T06:55:24,463 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-28T06:55:24,463 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732776924463"}]},"ts":"9223372036854775807"} 2024-11-28T06:55:24,465 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-28T06:55:24,466 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=78, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T06:55:24,467 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 80 msec 2024-11-28T06:55:24,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-28T06:55:24,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-28T06:55:24,536 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:55:24,545 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248 (was 248), OpenFileDescriptor=449 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=466 (was 384) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8126 (was 7741) - AvailableMemoryMB LEAK? - 2024-11-28T06:55:24,553 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=466, ProcessCount=11, AvailableMemoryMB=8125 2024-11-28T06:55:24,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T06:55:24,555 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T06:55:24,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=79, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-28T06:55:24,557 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T06:55:24,557 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:55:24,557 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 79 2024-11-28T06:55:24,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-28T06:55:24,558 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T06:55:24,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742112_1288 (size=1061) 2024-11-28T06:55:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-28T06:55:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-28T06:55:24,969 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7 2024-11-28T06:55:24,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742113_1289 (size=53) 2024-11-28T06:55:25,168 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-28T06:55:25,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-28T06:55:25,378 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T06:55:25,378 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a23eb2f63cfcb5c9c85b72f46ebe560c, disabling compactions & flushes 2024-11-28T06:55:25,378 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:25,378 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:25,378 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. after waiting 0 ms 2024-11-28T06:55:25,378 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:25,378 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:25,378 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a23eb2f63cfcb5c9c85b72f46ebe560c: Waiting for close lock at 1732776925378Disabling compacts and flushes for region at 1732776925378Disabling writes for close at 1732776925378Writing region close event to WAL at 1732776925378Closed at 1732776925378 2024-11-28T06:55:25,380 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T06:55:25,380 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732776925380"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732776925380"}]},"ts":"1732776925380"} 2024-11-28T06:55:25,382 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T06:55:25,383 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T06:55:25,383 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732776925383"}]},"ts":"1732776925383"} 2024-11-28T06:55:25,385 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-28T06:55:25,386 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a23eb2f63cfcb5c9c85b72f46ebe560c, ASSIGN}] 2024-11-28T06:55:25,387 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a23eb2f63cfcb5c9c85b72f46ebe560c, ASSIGN 2024-11-28T06:55:25,388 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a23eb2f63cfcb5c9c85b72f46ebe560c, ASSIGN; state=OFFLINE, location=1cb9cf437a36,43341,1732776825568; forceNewPlan=false, retain=false 2024-11-28T06:55:25,539 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=a23eb2f63cfcb5c9c85b72f46ebe560c, regionState=OPENING, regionLocation=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:25,543 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a23eb2f63cfcb5c9c85b72f46ebe560c, ASSIGN because future has completed 2024-11-28T06:55:25,544 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; OpenRegionProcedure a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568}] 2024-11-28T06:55:25,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-28T06:55:25,700 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:25,700 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} 2024-11-28T06:55:25,701 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:25,701 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T06:55:25,701 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:25,701 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:25,702 INFO [StoreOpener-a23eb2f63cfcb5c9c85b72f46ebe560c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:25,703 INFO [StoreOpener-a23eb2f63cfcb5c9c85b72f46ebe560c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T06:55:25,704 INFO [StoreOpener-a23eb2f63cfcb5c9c85b72f46ebe560c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a23eb2f63cfcb5c9c85b72f46ebe560c columnFamilyName A 2024-11-28T06:55:25,704 DEBUG [StoreOpener-a23eb2f63cfcb5c9c85b72f46ebe560c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:55:25,704 INFO [StoreOpener-a23eb2f63cfcb5c9c85b72f46ebe560c-1 {}] regionserver.HStore(327): Store=a23eb2f63cfcb5c9c85b72f46ebe560c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:55:25,704 INFO [StoreOpener-a23eb2f63cfcb5c9c85b72f46ebe560c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:25,705 INFO [StoreOpener-a23eb2f63cfcb5c9c85b72f46ebe560c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T06:55:25,705 INFO [StoreOpener-a23eb2f63cfcb5c9c85b72f46ebe560c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a23eb2f63cfcb5c9c85b72f46ebe560c columnFamilyName B 2024-11-28T06:55:25,705 DEBUG [StoreOpener-a23eb2f63cfcb5c9c85b72f46ebe560c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:55:25,706 INFO [StoreOpener-a23eb2f63cfcb5c9c85b72f46ebe560c-1 {}] regionserver.HStore(327): Store=a23eb2f63cfcb5c9c85b72f46ebe560c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:55:25,706 INFO [StoreOpener-a23eb2f63cfcb5c9c85b72f46ebe560c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:25,707 INFO [StoreOpener-a23eb2f63cfcb5c9c85b72f46ebe560c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T06:55:25,707 INFO [StoreOpener-a23eb2f63cfcb5c9c85b72f46ebe560c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a23eb2f63cfcb5c9c85b72f46ebe560c columnFamilyName C 2024-11-28T06:55:25,707 DEBUG [StoreOpener-a23eb2f63cfcb5c9c85b72f46ebe560c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:55:25,707 INFO [StoreOpener-a23eb2f63cfcb5c9c85b72f46ebe560c-1 {}] regionserver.HStore(327): Store=a23eb2f63cfcb5c9c85b72f46ebe560c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:55:25,708 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:25,708 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:25,708 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:25,708 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:25,709 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:25,709 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:25,709 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T06:55:25,710 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:25,712 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T06:55:25,712 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened a23eb2f63cfcb5c9c85b72f46ebe560c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63046643, jitterRate=-0.06053180992603302}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T06:55:25,712 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:25,713 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for a23eb2f63cfcb5c9c85b72f46ebe560c: Running coprocessor pre-open hook at 1732776925701Writing region info on filesystem at 1732776925701Initializing all the Stores at 1732776925702 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732776925702Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732776925702Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732776925702Cleaning up temporary data from old regions at 1732776925709 (+7 ms)Running coprocessor post-open hooks at 1732776925712 (+3 ms)Region opened successfully at 1732776925713 (+1 ms) 2024-11-28T06:55:25,714 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., pid=81, masterSystemTime=1732776925697 2024-11-28T06:55:25,716 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:25,716 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:25,716 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=a23eb2f63cfcb5c9c85b72f46ebe560c, regionState=OPEN, openSeqNum=2, regionLocation=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:25,718 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=80, state=RUNNABLE, hasLock=false; OpenRegionProcedure a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 because future has completed 2024-11-28T06:55:25,721 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-28T06:55:25,721 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; OpenRegionProcedure a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 in 175 msec 2024-11-28T06:55:25,723 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-11-28T06:55:25,723 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a23eb2f63cfcb5c9c85b72f46ebe560c, ASSIGN in 335 msec 2024-11-28T06:55:25,724 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T06:55:25,724 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732776925724"}]},"ts":"1732776925724"} 2024-11-28T06:55:25,725 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-28T06:55:25,726 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T06:55:25,727 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1710 sec 2024-11-28T06:55:26,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-28T06:55:26,706 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:55:26,708 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@197fefe0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:26,708 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:55:26,708 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:55:26,710 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:55:26,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:55:26,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:55:26,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@580a5472, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:26,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:55:26,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:55:26,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:26,712 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60276, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:55:26,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3392523c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:26,714 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:55:26,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:55:26,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:55:26,717 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40010, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:55:26,718 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T06:55:26,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 1cb9cf437a36,37503,1732776824789 2024-11-28T06:55:26,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4de816b4 2024-11-28T06:55:26,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T06:55:26,720 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60292, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T06:55:26,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T06:55:26,723 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T06:55:26,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-28T06:55:26,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742114_1290 (size=1097) 2024-11-28T06:55:27,137 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-28T06:55:27,137 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-28T06:55:27,140 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T06:55:27,143 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a23eb2f63cfcb5c9c85b72f46ebe560c, REOPEN/MOVE}] 2024-11-28T06:55:27,143 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a23eb2f63cfcb5c9c85b72f46ebe560c, REOPEN/MOVE 2024-11-28T06:55:27,144 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=a23eb2f63cfcb5c9c85b72f46ebe560c, regionState=CLOSING, regionLocation=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:27,146 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a23eb2f63cfcb5c9c85b72f46ebe560c, REOPEN/MOVE because future has completed 2024-11-28T06:55:27,146 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T06:55:27,146 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568}] 2024-11-28T06:55:27,299 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:27,299 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T06:55:27,299 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing a23eb2f63cfcb5c9c85b72f46ebe560c, disabling compactions & flushes 2024-11-28T06:55:27,299 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:27,299 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:27,299 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. after waiting 0 ms 2024-11-28T06:55:27,299 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:27,303 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-28T06:55:27,304 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:27,304 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for a23eb2f63cfcb5c9c85b72f46ebe560c: Waiting for close lock at 1732776927299Running coprocessor pre-close hooks at 1732776927299Disabling compacts and flushes for region at 1732776927299Disabling writes for close at 1732776927299Writing region close event to WAL at 1732776927299Running coprocessor post-close hooks at 1732776927304 (+5 ms)Closed at 1732776927304 2024-11-28T06:55:27,304 WARN [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionServer(3299): Not adding moved region record: a23eb2f63cfcb5c9c85b72f46ebe560c to self. 2024-11-28T06:55:27,306 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:27,306 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=a23eb2f63cfcb5c9c85b72f46ebe560c, regionState=CLOSED 2024-11-28T06:55:27,307 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 because future has completed 2024-11-28T06:55:27,309 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-28T06:55:27,309 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 in 162 msec 2024-11-28T06:55:27,310 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a23eb2f63cfcb5c9c85b72f46ebe560c, REOPEN/MOVE; state=CLOSED, location=1cb9cf437a36,43341,1732776825568; forceNewPlan=false, retain=true 2024-11-28T06:55:27,461 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=a23eb2f63cfcb5c9c85b72f46ebe560c, regionState=OPENING, regionLocation=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:27,463 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a23eb2f63cfcb5c9c85b72f46ebe560c, REOPEN/MOVE because future has completed 2024-11-28T06:55:27,464 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568}] 2024-11-28T06:55:27,622 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:27,623 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7752): Opening region: {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} 2024-11-28T06:55:27,623 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:27,623 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T06:55:27,623 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7794): checking encryption for a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:27,623 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7797): checking classloading for a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:27,626 INFO [StoreOpener-a23eb2f63cfcb5c9c85b72f46ebe560c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:27,628 INFO [StoreOpener-a23eb2f63cfcb5c9c85b72f46ebe560c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T06:55:27,628 INFO [StoreOpener-a23eb2f63cfcb5c9c85b72f46ebe560c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a23eb2f63cfcb5c9c85b72f46ebe560c columnFamilyName A 2024-11-28T06:55:27,629 DEBUG [StoreOpener-a23eb2f63cfcb5c9c85b72f46ebe560c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:55:27,630 INFO [StoreOpener-a23eb2f63cfcb5c9c85b72f46ebe560c-1 {}] regionserver.HStore(327): Store=a23eb2f63cfcb5c9c85b72f46ebe560c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:55:27,631 INFO [StoreOpener-a23eb2f63cfcb5c9c85b72f46ebe560c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:27,631 INFO [StoreOpener-a23eb2f63cfcb5c9c85b72f46ebe560c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T06:55:27,632 INFO [StoreOpener-a23eb2f63cfcb5c9c85b72f46ebe560c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a23eb2f63cfcb5c9c85b72f46ebe560c columnFamilyName B 2024-11-28T06:55:27,632 DEBUG [StoreOpener-a23eb2f63cfcb5c9c85b72f46ebe560c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:55:27,632 INFO [StoreOpener-a23eb2f63cfcb5c9c85b72f46ebe560c-1 {}] regionserver.HStore(327): Store=a23eb2f63cfcb5c9c85b72f46ebe560c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:55:27,632 INFO [StoreOpener-a23eb2f63cfcb5c9c85b72f46ebe560c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:27,633 INFO [StoreOpener-a23eb2f63cfcb5c9c85b72f46ebe560c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T06:55:27,634 INFO [StoreOpener-a23eb2f63cfcb5c9c85b72f46ebe560c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a23eb2f63cfcb5c9c85b72f46ebe560c columnFamilyName C 2024-11-28T06:55:27,634 DEBUG [StoreOpener-a23eb2f63cfcb5c9c85b72f46ebe560c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:55:27,634 INFO [StoreOpener-a23eb2f63cfcb5c9c85b72f46ebe560c-1 {}] regionserver.HStore(327): Store=a23eb2f63cfcb5c9c85b72f46ebe560c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:55:27,634 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:27,635 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1038): replaying wal for a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:27,636 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:27,637 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:27,639 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1048): stopping wal replay for a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:27,639 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1060): Cleaning up temporary data for a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:27,639 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T06:55:27,640 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1093): writing seq id for a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:27,641 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1114): Opened a23eb2f63cfcb5c9c85b72f46ebe560c; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71124616, jitterRate=0.05983936786651611}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T06:55:27,641 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:27,641 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1006): Region open journal for a23eb2f63cfcb5c9c85b72f46ebe560c: Running coprocessor pre-open hook at 1732776927624Writing region info on filesystem at 1732776927624Initializing all the Stores at 1732776927625 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732776927625Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732776927626 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732776927626Cleaning up temporary data from old regions at 1732776927639 (+13 ms)Running coprocessor post-open hooks at 1732776927641 (+2 ms)Region opened successfully at 1732776927641 2024-11-28T06:55:27,642 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., pid=86, masterSystemTime=1732776927617 2024-11-28T06:55:27,644 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:27,644 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:27,644 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=a23eb2f63cfcb5c9c85b72f46ebe560c, regionState=OPEN, openSeqNum=5, regionLocation=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:27,646 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 because future has completed 2024-11-28T06:55:27,648 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=84 2024-11-28T06:55:27,648 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=84, state=SUCCESS, hasLock=false; OpenRegionProcedure a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 in 182 msec 2024-11-28T06:55:27,649 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-28T06:55:27,650 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a23eb2f63cfcb5c9c85b72f46ebe560c, REOPEN/MOVE in 505 msec 2024-11-28T06:55:27,652 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-28T06:55:27,652 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 510 msec 2024-11-28T06:55:27,653 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 929 msec 2024-11-28T06:55:27,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-28T06:55:27,654 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:55:27,655 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b2f185f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:27,655 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:55:27,655 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:55:27,656 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:55:27,656 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:55:27,656 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:55:27,656 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@592721b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:27,656 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:55:27,656 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:55:27,656 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:27,657 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60318, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:55:27,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4791ae54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:27,658 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a739f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:27,658 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:55:27,658 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:55:27,658 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:55:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:55:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:55:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f04fa05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:55:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:55:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:27,660 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:55:27,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74430f4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:27,660 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@87fd48f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:27,660 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:55:27,660 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:55:27,661 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:55:27,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:55:27,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:55:27,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b84325b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:27,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:55:27,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:55:27,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:27,662 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60354, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:55:27,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dc481f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:27,663 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0bd6a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:27,663 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:55:27,663 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:55:27,664 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:55:27,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:55:27,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:55:27,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e62a4c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:27,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:55:27,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:55:27,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:27,665 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60380, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:55:27,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@638dcadc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:27,666 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72c86b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:27,666 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:55:27,666 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:55:27,666 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:55:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:55:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:55:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6580cfe9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:55:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:55:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:27,668 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60388, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:55:27,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@529b1c2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:27,669 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d098cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:27,669 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:55:27,669 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:55:27,670 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:55:27,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:55:27,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:55:27,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a7f6650, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:27,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:55:27,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:55:27,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:27,671 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60402, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:55:27,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fe92efa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:27,672 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c9deaeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:27,672 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:55:27,672 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:55:27,672 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:55:27,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:55:27,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:55:27,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9d40e2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:27,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:55:27,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:55:27,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:27,673 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60414, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:55:27,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c9b882e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:27,674 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f8ca2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:27,674 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:55:27,674 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:55:27,675 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:55:27,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:55:27,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:55:27,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11ad5415, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:27,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:55:27,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:55:27,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:27,676 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60432, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:55:27,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a19380e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:27,678 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1515ed5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:27,678 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:55:27,678 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:55:27,679 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:55:27,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:55:27,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:55:27,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bbc5688, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:27,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:55:27,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:55:27,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:27,680 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60452, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:55:27,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@125c9c3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:27,682 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d10c554, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:27,682 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:55:27,682 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:55:27,683 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:55:27,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:55:27,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:55:27,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40d7bfa7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:27,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:55:27,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:55:27,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:27,685 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60482, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:55:27,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69678449, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:27,686 DEBUG [Thread-1369 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:55:27,686 DEBUG [Thread-1367 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:55:27,686 DEBUG [Thread-1356 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:55:27,686 DEBUG [Thread-1362 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:55:27,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:55:27,687 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:55:27,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:55:27,687 DEBUG [Thread-1365 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:55:27,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:55:27,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:55:27,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:55:27,687 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:55:27,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:55:27,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:55:27,688 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40022, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:55:27,688 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:55:27,688 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:55:27,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:55:27,689 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40044, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:55:27,689 DEBUG [Thread-1373 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:55:27,690 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:55:27,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:55:27,690 DEBUG [Thread-1360 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:55:27,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:55:27,690 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:55:27,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5] 2024-11-28T06:55:27,691 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40070, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:55:27,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:55:27,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees 2024-11-28T06:55:27,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:55:27,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5] 2024-11-28T06:55:27,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-28T06:55:27,693 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:55:27,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5] 2024-11-28T06:55:27,694 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40080, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:55:27,694 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:55:27,695 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:55:27,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5] 2024-11-28T06:55:27,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5] 2024-11-28T06:55:27,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5] 2024-11-28T06:55:27,697 DEBUG [Thread-1354 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:55:27,697 DEBUG [Thread-1358 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:55:27,697 DEBUG [Thread-1354 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:55:27,698 DEBUG [Thread-1354 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:55:27,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:55:27,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:55:27,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5] 2024-11-28T06:55:27,699 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40084, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:55:27,699 DEBUG [Thread-1371 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:55:27,700 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40092, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:55:27,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:55:27,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:55:27,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5] 2024-11-28T06:55:27,702 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40098, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:55:27,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5] 2024-11-28T06:55:27,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5] 2024-11-28T06:55:27,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:27,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a23eb2f63cfcb5c9c85b72f46ebe560c 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T06:55:27,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=A 2024-11-28T06:55:27,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:27,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=B 2024-11-28T06:55:27,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:27,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=C 2024-11-28T06:55:27,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:27,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128fac978cbfeeb4ddd948a705d752f0ffa_a23eb2f63cfcb5c9c85b72f46ebe560c is 50, key is test_row_0/A:col10/1732776927696/Put/seqid=0 2024-11-28T06:55:27,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742115_1291 (size=19474) 2024-11-28T06:55:27,767 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:55:27,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732776927731, max timestamp=1732776927766Num uniques 859; 2024-11-28T06:55:27,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-28T06:55:27,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732776927731, max timestamp=1732776927766Num uniques 859; 2024-11-28T06:55:27,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-28T06:55:27,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732776927731, max timestamp=1732776927766Num uniques 859; 2024-11-28T06:55:27,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-28T06:55:27,775 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128fac978cbfeeb4ddd948a705d752f0ffa_a23eb2f63cfcb5c9c85b72f46ebe560c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128fac978cbfeeb4ddd948a705d752f0ffa_a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:27,776 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/edf4cd3943584a03bfd7325972857f59, store: [table=TestAcidGuarantees family=A region=a23eb2f63cfcb5c9c85b72f46ebe560c] 2024-11-28T06:55:27,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/edf4cd3943584a03bfd7325972857f59 is 175, key is test_row_0/A:col10/1732776927696/Put/seqid=0 2024-11-28T06:55:27,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742116_1292 (size=56733) 2024-11-28T06:55:27,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-28T06:55:27,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:27,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40084 deadline: 1732776987796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:27,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:27,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:27,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:27,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:27,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732776987812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:27,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:27,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:27,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:27,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40022 deadline: 1732776987812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:27,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:27,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:27,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:27,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:27,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:27,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1732776987812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:27,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:27,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:27,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:27,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:27,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732776987814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:27,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:27,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:27,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:27,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-28T06:55:27,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:27,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. as already flushing 2024-11-28T06:55:27,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:27,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:27,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:27,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:27,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:27,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40084 deadline: 1732776987916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:27,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:27,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:27,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:27,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:27,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732776987925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:27,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:27,937 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:27,937 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:27,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40022 deadline: 1732776987925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:27,937 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:27,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:27,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:27,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:27,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1732776987925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:27,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:27,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:27,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:27,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:27,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:27,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732776987936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:27,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:27,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:27,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:28,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-28T06:55:28,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:28,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. as already flushing 2024-11-28T06:55:28,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:28,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:28,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:28,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:28,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-28T06:55:28,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:28,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40084 deadline: 1732776988135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:28,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:28,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:28,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:28,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:28,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732776988145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:28,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:28,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:28,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:28,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1732776988146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:28,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:28,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:28,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:28,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:28,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:28,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40022 deadline: 1732776988146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:28,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:28,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:28,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:28,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-28T06:55:28,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:28,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. as already flushing 2024-11-28T06:55:28,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:28,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:28,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:28,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:28,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:28,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732776988155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:28,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:28,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:28,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:28,183 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/edf4cd3943584a03bfd7325972857f59 2024-11-28T06:55:28,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/6260b1ba85dd4d18babcb0235574f5a7 is 50, key is test_row_0/B:col10/1732776927696/Put/seqid=0 2024-11-28T06:55:28,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742117_1293 (size=12001) 2024-11-28T06:55:28,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/6260b1ba85dd4d18babcb0235574f5a7 2024-11-28T06:55:28,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/3c6f4b1fd2d448ed90cd26cd867bc487 is 50, key is test_row_0/C:col10/1732776927696/Put/seqid=0 2024-11-28T06:55:28,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742118_1294 (size=12001) 2024-11-28T06:55:28,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/3c6f4b1fd2d448ed90cd26cd867bc487 2024-11-28T06:55:28,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/edf4cd3943584a03bfd7325972857f59 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/edf4cd3943584a03bfd7325972857f59 2024-11-28T06:55:28,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/edf4cd3943584a03bfd7325972857f59, entries=300, sequenceid=15, filesize=55.4 K 2024-11-28T06:55:28,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/6260b1ba85dd4d18babcb0235574f5a7 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/6260b1ba85dd4d18babcb0235574f5a7 2024-11-28T06:55:28,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/6260b1ba85dd4d18babcb0235574f5a7, entries=150, sequenceid=15, filesize=11.7 K 2024-11-28T06:55:28,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/3c6f4b1fd2d448ed90cd26cd867bc487 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/3c6f4b1fd2d448ed90cd26cd867bc487 2024-11-28T06:55:28,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-28T06:55:28,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:28,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. as already flushing 2024-11-28T06:55:28,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:28,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:28,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:28,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:28,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/3c6f4b1fd2d448ed90cd26cd867bc487, entries=150, sequenceid=15, filesize=11.7 K 2024-11-28T06:55:28,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-28T06:55:28,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for a23eb2f63cfcb5c9c85b72f46ebe560c in 607ms, sequenceid=15, compaction requested=false 2024-11-28T06:55:28,317 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-28T06:55:28,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a23eb2f63cfcb5c9c85b72f46ebe560c: 2024-11-28T06:55:28,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-28T06:55:28,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:28,464 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2902): Flushing a23eb2f63cfcb5c9c85b72f46ebe560c 3/3 column families, dataSize=134.18 KB heapSize=366.23 KB 2024-11-28T06:55:28,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. as already flushing 2024-11-28T06:55:28,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=A 2024-11-28T06:55:28,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:28,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:55:28,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=B 2024-11-28T06:55:28,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:55:28,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=C 2024-11-28T06:55:28,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:55:28,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128dd91e6966a3d4334ad8b13da961f2da6_a23eb2f63cfcb5c9c85b72f46ebe560c is 50, key is test_row_0/A:col10/1732776928463/Put/seqid=0 2024-11-28T06:55:28,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:28,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40084 deadline: 1732776988476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:28,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:28,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:28,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:28,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:28,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732776988477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:28,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:28,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:28,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:28,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:28,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732776988481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:28,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:28,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:28,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:28,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40022 deadline: 1732776988483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:28,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:28,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:28,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:28,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:28,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:28,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1732776988495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:28,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:28,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:28,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:28,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742119_1295 (size=14594) 2024-11-28T06:55:28,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40084 deadline: 1732776988605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:28,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:28,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:28,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:28,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1732776988606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:28,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:28,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:28,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:28,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732776988606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:28,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:28,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:28,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:28,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40022 deadline: 1732776988606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:28,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:28,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:28,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:28,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-28T06:55:28,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:28,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40084 deadline: 1732776988816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:28,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:28,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:28,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:28,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:28,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40022 deadline: 1732776988826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:28,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:28,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1732776988826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:28,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:28,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:28,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:28,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732776988826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:28,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:28,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:28,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:28,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:28,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:28,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:28,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:28,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:55:28,914 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128dd91e6966a3d4334ad8b13da961f2da6_a23eb2f63cfcb5c9c85b72f46ebe560c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128dd91e6966a3d4334ad8b13da961f2da6_a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:28,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/7595faaf5e04498f8ea538614292f7d8, store: [table=TestAcidGuarantees family=A region=a23eb2f63cfcb5c9c85b72f46ebe560c] 2024-11-28T06:55:28,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/7595faaf5e04498f8ea538614292f7d8 is 175, key is test_row_0/A:col10/1732776928463/Put/seqid=0 2024-11-28T06:55:28,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742120_1296 (size=39549) 2024-11-28T06:55:28,925 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/7595faaf5e04498f8ea538614292f7d8 2024-11-28T06:55:28,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/28e4614c6a874c309e512a7bc7ef84a8 is 50, key is test_row_0/B:col10/1732776928463/Put/seqid=0 2024-11-28T06:55:28,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742121_1297 (size=12001) 2024-11-28T06:55:29,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:29,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732776989010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:29,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:29,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:29,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:29,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40084 deadline: 1732776989136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:29,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:29,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:29,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:29,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:29,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1732776989147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:29,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:29,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40022 deadline: 1732776989148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:29,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:29,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:29,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732776989149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:29,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:29,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:29,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:29,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:29,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:29,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:29,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:29,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:29,268 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T06:55:29,343 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/28e4614c6a874c309e512a7bc7ef84a8 2024-11-28T06:55:29,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/7a2fa727256143189af7fa469c31de7e is 50, key is test_row_0/C:col10/1732776928463/Put/seqid=0 2024-11-28T06:55:29,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742122_1298 (size=12001) 2024-11-28T06:55:29,379 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/7a2fa727256143189af7fa469c31de7e 2024-11-28T06:55:29,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/7595faaf5e04498f8ea538614292f7d8 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/7595faaf5e04498f8ea538614292f7d8 2024-11-28T06:55:29,392 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/7595faaf5e04498f8ea538614292f7d8, entries=200, sequenceid=49, filesize=38.6 K 2024-11-28T06:55:29,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/28e4614c6a874c309e512a7bc7ef84a8 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/28e4614c6a874c309e512a7bc7ef84a8 2024-11-28T06:55:29,399 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/28e4614c6a874c309e512a7bc7ef84a8, entries=150, sequenceid=49, filesize=11.7 K 2024-11-28T06:55:29,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/7a2fa727256143189af7fa469c31de7e as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/7a2fa727256143189af7fa469c31de7e 2024-11-28T06:55:29,407 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/7a2fa727256143189af7fa469c31de7e, entries=150, sequenceid=49, filesize=11.7 K 2024-11-28T06:55:29,408 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~386.11 KB/395376, currentSize=46.96 KB/48090 for a23eb2f63cfcb5c9c85b72f46ebe560c in 944ms, sequenceid=49, compaction requested=false 2024-11-28T06:55:29,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2603): Flush status journal for a23eb2f63cfcb5c9c85b72f46ebe560c: 2024-11-28T06:55:29,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:29,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88 2024-11-28T06:55:29,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=88 2024-11-28T06:55:29,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-28T06:55:29,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7150 sec 2024-11-28T06:55:29,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees in 1.7230 sec 2024-11-28T06:55:29,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:29,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a23eb2f63cfcb5c9c85b72f46ebe560c 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T06:55:29,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=A 2024-11-28T06:55:29,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:29,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=B 2024-11-28T06:55:29,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:29,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=C 2024-11-28T06:55:29,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:29,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128e0002ce346b7407797403e9d5bf47393_a23eb2f63cfcb5c9c85b72f46ebe560c is 50, key is test_row_0/A:col10/1732776929659/Put/seqid=0 2024-11-28T06:55:29,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742123_1299 (size=14594) 2024-11-28T06:55:29,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:29,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732776989715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:29,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:29,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:29,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:29,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732776929696, max timestamp=1732776929723Num uniques 859; 2024-11-28T06:55:29,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T06:55:29,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732776929696, max timestamp=1732776929723Num uniques 859; 2024-11-28T06:55:29,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T06:55:29,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732776929696, max timestamp=1732776929723Num uniques 859; 2024-11-28T06:55:29,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T06:55:29,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:29,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1732776989793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:29,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:29,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:29,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:29,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:29,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40084 deadline: 1732776989808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:29,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:29,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:29,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:29,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:29,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40022 deadline: 1732776989809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:29,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:29,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:29,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:29,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-28T06:55:29,836 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:55:29,836 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:55:29,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-11-28T06:55:29,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-28T06:55:29,839 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:55:29,840 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:55:29,840 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:55:29,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:29,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732776989837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:29,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:29,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:29,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:29,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:29,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40084 deadline: 1732776989925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:29,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:29,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:29,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1732776989925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:29,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:29,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:29,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:29,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:29,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:29,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-28T06:55:29,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:29,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40022 deadline: 1732776989936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:29,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:29,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:29,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:29,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-28T06:55:29,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:29,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. as already flushing 2024-11-28T06:55:29,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:29,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:29,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:29,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:30,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:30,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732776990026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:30,036 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_2' in a23eb2f63cfcb5c9c85b72f46ebe560c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:55:30,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:30,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:30,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:30,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:30,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732776990056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:30,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:30,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:30,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:30,083 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:55:30,088 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128e0002ce346b7407797403e9d5bf47393_a23eb2f63cfcb5c9c85b72f46ebe560c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e0002ce346b7407797403e9d5bf47393_a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:30,089 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/3e115d0567324cdab8cb58fa806836ae, store: [table=TestAcidGuarantees family=A region=a23eb2f63cfcb5c9c85b72f46ebe560c] 2024-11-28T06:55:30,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/3e115d0567324cdab8cb58fa806836ae is 175, key is test_row_0/A:col10/1732776929659/Put/seqid=0 2024-11-28T06:55:30,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742124_1300 (size=39549) 2024-11-28T06:55:30,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-28T06:55:30,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:30,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. as already flushing 2024-11-28T06:55:30,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:30,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:30,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:30,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:30,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-28T06:55:30,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:30,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1732776990146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:30,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:30,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40084 deadline: 1732776990146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:30,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:30,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:30,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:30,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:30,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:30,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:30,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:30,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40022 deadline: 1732776990158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:30,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:30,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:30,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:30,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-28T06:55:30,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:30,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. as already flushing 2024-11-28T06:55:30,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:30,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:30,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:30,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:30,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:30,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732776990375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:30,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:30,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:30,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:30,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-28T06:55:30,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:30,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. as already flushing 2024-11-28T06:55:30,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:30,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:30,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:30,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:30,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-28T06:55:30,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:30,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1732776990466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:30,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:30,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:30,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:30,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:30,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40084 deadline: 1732776990476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:30,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:30,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:30,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:30,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:30,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40022 deadline: 1732776990485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:30,538 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/3e115d0567324cdab8cb58fa806836ae 2024-11-28T06:55:30,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/b3526ad723c5451dba69debebcd36600 is 50, key is test_row_0/B:col10/1732776929659/Put/seqid=0 2024-11-28T06:55:30,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742125_1301 (size=12001) 2024-11-28T06:55:30,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-28T06:55:30,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:30,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. as already flushing 2024-11-28T06:55:30,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:30,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:30,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:30,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:30,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-28T06:55:30,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:30,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. as already flushing 2024-11-28T06:55:30,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:30,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:30,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:30,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:30,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:30,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732776990895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:30,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:30,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:30,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:30,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-28T06:55:30,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:30,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. as already flushing 2024-11-28T06:55:30,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:30,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:30,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:30,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:30,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-28T06:55:30,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/b3526ad723c5451dba69debebcd36600 2024-11-28T06:55:30,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:30,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1732776990985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:30,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:30,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:30,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:31,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/e75f3b52251d40e38219183f5a4e5590 is 50, key is test_row_0/C:col10/1732776929659/Put/seqid=0 2024-11-28T06:55:31,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:31,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40022 deadline: 1732776990996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:31,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:31,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:31,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40084 deadline: 1732776990996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:31,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:31,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:31,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:31,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:31,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:31,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742126_1302 (size=12001) 2024-11-28T06:55:31,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-28T06:55:31,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:31,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. as already flushing 2024-11-28T06:55:31,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:31,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:31,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:31,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-28T06:55:31,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:31,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. as already flushing 2024-11-28T06:55:31,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:31,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:31,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:31,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:31,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-28T06:55:31,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:31,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. as already flushing 2024-11-28T06:55:31,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:31,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:31,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:31,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:31,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/e75f3b52251d40e38219183f5a4e5590 2024-11-28T06:55:31,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/3e115d0567324cdab8cb58fa806836ae as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/3e115d0567324cdab8cb58fa806836ae 2024-11-28T06:55:31,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/3e115d0567324cdab8cb58fa806836ae, entries=200, sequenceid=60, filesize=38.6 K 2024-11-28T06:55:31,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/b3526ad723c5451dba69debebcd36600 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/b3526ad723c5451dba69debebcd36600 2024-11-28T06:55:31,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/b3526ad723c5451dba69debebcd36600, entries=150, sequenceid=60, filesize=11.7 K 2024-11-28T06:55:31,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/e75f3b52251d40e38219183f5a4e5590 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/e75f3b52251d40e38219183f5a4e5590 2024-11-28T06:55:31,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/e75f3b52251d40e38219183f5a4e5590, entries=150, sequenceid=60, filesize=11.7 K 2024-11-28T06:55:31,443 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for a23eb2f63cfcb5c9c85b72f46ebe560c in 1782ms, sequenceid=60, compaction requested=true 2024-11-28T06:55:31,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a23eb2f63cfcb5c9c85b72f46ebe560c: 2024-11-28T06:55:31,443 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:55:31,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a23eb2f63cfcb5c9c85b72f46ebe560c:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:55:31,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:31,444 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:55:31,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a23eb2f63cfcb5c9c85b72f46ebe560c:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:55:31,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:31,445 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 135831 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:55:31,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a23eb2f63cfcb5c9c85b72f46ebe560c:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:55:31,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:55:31,445 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): a23eb2f63cfcb5c9c85b72f46ebe560c/A is initiating minor compaction (all files) 2024-11-28T06:55:31,445 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a23eb2f63cfcb5c9c85b72f46ebe560c/A in TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:31,445 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/edf4cd3943584a03bfd7325972857f59, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/7595faaf5e04498f8ea538614292f7d8, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/3e115d0567324cdab8cb58fa806836ae] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp, totalSize=132.6 K 2024-11-28T06:55:31,445 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:31,445 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. files: [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/edf4cd3943584a03bfd7325972857f59, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/7595faaf5e04498f8ea538614292f7d8, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/3e115d0567324cdab8cb58fa806836ae] 2024-11-28T06:55:31,445 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:55:31,445 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting edf4cd3943584a03bfd7325972857f59, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732776927692 2024-11-28T06:55:31,446 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): a23eb2f63cfcb5c9c85b72f46ebe560c/B is initiating minor compaction (all files) 2024-11-28T06:55:31,446 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a23eb2f63cfcb5c9c85b72f46ebe560c/B in TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:31,446 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/6260b1ba85dd4d18babcb0235574f5a7, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/28e4614c6a874c309e512a7bc7ef84a8, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/b3526ad723c5451dba69debebcd36600] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp, totalSize=35.2 K 2024-11-28T06:55:31,446 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7595faaf5e04498f8ea538614292f7d8, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732776927794 2024-11-28T06:55:31,446 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 6260b1ba85dd4d18babcb0235574f5a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732776927692 2024-11-28T06:55:31,446 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e115d0567324cdab8cb58fa806836ae, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732776928474 2024-11-28T06:55:31,450 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 28e4614c6a874c309e512a7bc7ef84a8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732776927811 2024-11-28T06:55:31,451 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting b3526ad723c5451dba69debebcd36600, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732776928478 2024-11-28T06:55:31,455 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a23eb2f63cfcb5c9c85b72f46ebe560c] 2024-11-28T06:55:31,459 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128cea6e16add504af4b615ca65dedf0139_a23eb2f63cfcb5c9c85b72f46ebe560c store=[table=TestAcidGuarantees family=A region=a23eb2f63cfcb5c9c85b72f46ebe560c] 2024-11-28T06:55:31,461 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128cea6e16add504af4b615ca65dedf0139_a23eb2f63cfcb5c9c85b72f46ebe560c, store=[table=TestAcidGuarantees family=A region=a23eb2f63cfcb5c9c85b72f46ebe560c] 2024-11-28T06:55:31,461 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128cea6e16add504af4b615ca65dedf0139_a23eb2f63cfcb5c9c85b72f46ebe560c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a23eb2f63cfcb5c9c85b72f46ebe560c] 2024-11-28T06:55:31,467 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a23eb2f63cfcb5c9c85b72f46ebe560c#B#compaction#257 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:55:31,467 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/b00c3b10e7f343e4bc8541d5d2671fb0 is 50, key is test_row_0/B:col10/1732776929659/Put/seqid=0 2024-11-28T06:55:31,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742127_1303 (size=4469) 2024-11-28T06:55:31,511 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a23eb2f63cfcb5c9c85b72f46ebe560c#A#compaction#256 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:55:31,511 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/631488c4571b4a18a7dd1c7db98ecea5 is 175, key is test_row_0/A:col10/1732776929659/Put/seqid=0 2024-11-28T06:55:31,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742128_1304 (size=12104) 2024-11-28T06:55:31,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742129_1305 (size=31058) 2024-11-28T06:55:31,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-28T06:55:31,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:31,540 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing a23eb2f63cfcb5c9c85b72f46ebe560c 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-28T06:55:31,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=A 2024-11-28T06:55:31,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:55:31,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=B 2024-11-28T06:55:31,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:55:31,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=C 2024-11-28T06:55:31,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:55:31,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112808585c9d3c9b4496a1a02b483076b049_a23eb2f63cfcb5c9c85b72f46ebe560c is 50, key is test_row_0/A:col10/1732776929808/Put/seqid=0 2024-11-28T06:55:31,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742130_1306 (size=12154) 2024-11-28T06:55:31,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:55:31,582 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112808585c9d3c9b4496a1a02b483076b049_a23eb2f63cfcb5c9c85b72f46ebe560c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112808585c9d3c9b4496a1a02b483076b049_a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:31,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/a1087292fa2b47e08a212d6aaacc0e98, store: [table=TestAcidGuarantees family=A region=a23eb2f63cfcb5c9c85b72f46ebe560c] 2024-11-28T06:55:31,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/a1087292fa2b47e08a212d6aaacc0e98 is 175, key is test_row_0/A:col10/1732776929808/Put/seqid=0 2024-11-28T06:55:31,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742131_1307 (size=30955) 2024-11-28T06:55:31,601 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/a1087292fa2b47e08a212d6aaacc0e98 2024-11-28T06:55:31,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/f556c8fe93824216bfa2dd923d7fc35b is 50, key is test_row_0/B:col10/1732776929808/Put/seqid=0 2024-11-28T06:55:31,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742132_1308 (size=12001) 2024-11-28T06:55:31,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:31,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. as already flushing 2024-11-28T06:55:31,930 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/b00c3b10e7f343e4bc8541d5d2671fb0 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/b00c3b10e7f343e4bc8541d5d2671fb0 2024-11-28T06:55:31,933 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/631488c4571b4a18a7dd1c7db98ecea5 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/631488c4571b4a18a7dd1c7db98ecea5 2024-11-28T06:55:31,937 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a23eb2f63cfcb5c9c85b72f46ebe560c/B of a23eb2f63cfcb5c9c85b72f46ebe560c into b00c3b10e7f343e4bc8541d5d2671fb0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:55:31,937 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a23eb2f63cfcb5c9c85b72f46ebe560c: 2024-11-28T06:55:31,937 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., storeName=a23eb2f63cfcb5c9c85b72f46ebe560c/B, priority=13, startTime=1732776931444; duration=0sec 2024-11-28T06:55:31,938 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:55:31,938 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a23eb2f63cfcb5c9c85b72f46ebe560c:B 2024-11-28T06:55:31,938 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:55:31,939 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:55:31,939 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): a23eb2f63cfcb5c9c85b72f46ebe560c/C is initiating minor compaction (all files) 2024-11-28T06:55:31,939 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a23eb2f63cfcb5c9c85b72f46ebe560c/C in TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:31,939 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/3c6f4b1fd2d448ed90cd26cd867bc487, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/7a2fa727256143189af7fa469c31de7e, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/e75f3b52251d40e38219183f5a4e5590] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp, totalSize=35.2 K 2024-11-28T06:55:31,939 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a23eb2f63cfcb5c9c85b72f46ebe560c/A of a23eb2f63cfcb5c9c85b72f46ebe560c into 631488c4571b4a18a7dd1c7db98ecea5(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:55:31,939 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a23eb2f63cfcb5c9c85b72f46ebe560c: 2024-11-28T06:55:31,939 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., storeName=a23eb2f63cfcb5c9c85b72f46ebe560c/A, priority=13, startTime=1732776931443; duration=0sec 2024-11-28T06:55:31,939 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:31,939 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a23eb2f63cfcb5c9c85b72f46ebe560c:A 2024-11-28T06:55:31,940 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c6f4b1fd2d448ed90cd26cd867bc487, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732776927692 2024-11-28T06:55:31,940 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a2fa727256143189af7fa469c31de7e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732776927811 2024-11-28T06:55:31,941 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting e75f3b52251d40e38219183f5a4e5590, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732776928478 2024-11-28T06:55:31,948 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a23eb2f63cfcb5c9c85b72f46ebe560c#C#compaction#260 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:55:31,948 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/1be44fd891304e3dab7129938f193889 is 50, key is test_row_0/C:col10/1732776929659/Put/seqid=0 2024-11-28T06:55:31,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742133_1309 (size=12104) 2024-11-28T06:55:31,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-28T06:55:32,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:32,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732776992000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:32,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:32,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:32,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:32,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:32,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1732776992007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:32,021 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_2' in a23eb2f63cfcb5c9c85b72f46ebe560c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:55:32,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:32,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:32,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:32,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:32,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40022 deadline: 1732776992016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:32,026 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_2' in a23eb2f63cfcb5c9c85b72f46ebe560c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:55:32,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:32,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:32,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:32,032 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/f556c8fe93824216bfa2dd923d7fc35b 2024-11-28T06:55:32,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:32,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40084 deadline: 1732776992025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:32,040 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_1' in a23eb2f63cfcb5c9c85b72f46ebe560c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:55:32,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:32,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:32,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:32,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/44be8c6e67594af89b7f9216e130abd2 is 50, key is test_row_0/C:col10/1732776929808/Put/seqid=0 2024-11-28T06:55:32,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742134_1310 (size=12001) 2024-11-28T06:55:32,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:32,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732776992045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:32,060 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_2' in a23eb2f63cfcb5c9c85b72f46ebe560c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4247 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:55:32,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:32,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:32,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:32,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:32,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732776992115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:32,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:32,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732776992336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:32,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:32,362 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/1be44fd891304e3dab7129938f193889 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/1be44fd891304e3dab7129938f193889 2024-11-28T06:55:32,366 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a23eb2f63cfcb5c9c85b72f46ebe560c/C of a23eb2f63cfcb5c9c85b72f46ebe560c into 1be44fd891304e3dab7129938f193889(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:55:32,366 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a23eb2f63cfcb5c9c85b72f46ebe560c: 2024-11-28T06:55:32,366 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., storeName=a23eb2f63cfcb5c9c85b72f46ebe560c/C, priority=13, startTime=1732776931445; duration=0sec 2024-11-28T06:55:32,366 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:32,366 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a23eb2f63cfcb5c9c85b72f46ebe560c:C 2024-11-28T06:55:32,452 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/44be8c6e67594af89b7f9216e130abd2 2024-11-28T06:55:32,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/a1087292fa2b47e08a212d6aaacc0e98 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/a1087292fa2b47e08a212d6aaacc0e98 2024-11-28T06:55:32,459 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/a1087292fa2b47e08a212d6aaacc0e98, entries=150, sequenceid=95, filesize=30.2 K 2024-11-28T06:55:32,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/f556c8fe93824216bfa2dd923d7fc35b as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/f556c8fe93824216bfa2dd923d7fc35b 2024-11-28T06:55:32,462 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/f556c8fe93824216bfa2dd923d7fc35b, entries=150, sequenceid=95, filesize=11.7 K 2024-11-28T06:55:32,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/44be8c6e67594af89b7f9216e130abd2 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/44be8c6e67594af89b7f9216e130abd2 2024-11-28T06:55:32,467 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/44be8c6e67594af89b7f9216e130abd2, entries=150, sequenceid=95, filesize=11.7 K 2024-11-28T06:55:32,468 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for a23eb2f63cfcb5c9c85b72f46ebe560c in 929ms, sequenceid=95, compaction requested=false 2024-11-28T06:55:32,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for a23eb2f63cfcb5c9c85b72f46ebe560c: 2024-11-28T06:55:32,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:32,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-11-28T06:55:32,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-11-28T06:55:32,470 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-28T06:55:32,470 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6290 sec 2024-11-28T06:55:32,471 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 2.6350 sec 2024-11-28T06:55:32,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:32,659 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a23eb2f63cfcb5c9c85b72f46ebe560c 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-28T06:55:32,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=A 2024-11-28T06:55:32,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:32,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=B 2024-11-28T06:55:32,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:32,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=C 2024-11-28T06:55:32,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:32,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128763375aa6ab94be0bc0bb1177ab57367_a23eb2f63cfcb5c9c85b72f46ebe560c is 50, key is test_row_0/A:col10/1732776932658/Put/seqid=0 2024-11-28T06:55:32,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742135_1311 (size=14594) 2024-11-28T06:55:32,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:32,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732776992787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:32,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:32,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:32,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:32,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:32,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732776992895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:32,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:32,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:32,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:33,071 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:55:33,075 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128763375aa6ab94be0bc0bb1177ab57367_a23eb2f63cfcb5c9c85b72f46ebe560c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128763375aa6ab94be0bc0bb1177ab57367_a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:33,076 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/a38e26179add4c799199dc8fb82b6979, store: [table=TestAcidGuarantees family=A region=a23eb2f63cfcb5c9c85b72f46ebe560c] 2024-11-28T06:55:33,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/a38e26179add4c799199dc8fb82b6979 is 175, key is test_row_0/A:col10/1732776932658/Put/seqid=0 2024-11-28T06:55:33,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742136_1312 (size=39549) 2024-11-28T06:55:33,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:33,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732776993105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:33,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:33,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:33,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:33,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:33,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732776993426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:33,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:33,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:33,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:33,485 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=110, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/a38e26179add4c799199dc8fb82b6979 2024-11-28T06:55:33,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/cfed94df144d4d67ae259e63d864dcda is 50, key is test_row_0/B:col10/1732776932658/Put/seqid=0 2024-11-28T06:55:33,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742137_1313 (size=12001) 2024-11-28T06:55:33,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/cfed94df144d4d67ae259e63d864dcda 2024-11-28T06:55:33,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/cc55df6889634495b3df31ffea6392db is 50, key is test_row_0/C:col10/1732776932658/Put/seqid=0 2024-11-28T06:55:33,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742138_1314 (size=12001) 2024-11-28T06:55:33,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:33,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732776993945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:33,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:33,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:33,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:34,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-28T06:55:34,015 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:55:34,016 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:55:34,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-28T06:55:34,017 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:55:34,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-28T06:55:34,018 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:55:34,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:55:34,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:34,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1732776994035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:34,043 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_2' in a23eb2f63cfcb5c9c85b72f46ebe560c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4250 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:55:34,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:34,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:34,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:34,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:34,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40022 deadline: 1732776994056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:34,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:34,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40084 deadline: 1732776994056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:34,064 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_2' in a23eb2f63cfcb5c9c85b72f46ebe560c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4255 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:55:34,064 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_1' in a23eb2f63cfcb5c9c85b72f46ebe560c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4256 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:55:34,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:34,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:34,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:34,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:34,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:34,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:34,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-28T06:55:34,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-28T06:55:34,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:34,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. as already flushing 2024-11-28T06:55:34,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:34,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:34,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:34,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:34,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/cc55df6889634495b3df31ffea6392db 2024-11-28T06:55:34,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/a38e26179add4c799199dc8fb82b6979 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/a38e26179add4c799199dc8fb82b6979 2024-11-28T06:55:34,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/a38e26179add4c799199dc8fb82b6979, entries=200, sequenceid=110, filesize=38.6 K 2024-11-28T06:55:34,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-28T06:55:34,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:34,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. as already flushing 2024-11-28T06:55:34,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:34,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/cfed94df144d4d67ae259e63d864dcda as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/cfed94df144d4d67ae259e63d864dcda 2024-11-28T06:55:34,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:34,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:34,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:34,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/cfed94df144d4d67ae259e63d864dcda, entries=150, sequenceid=110, filesize=11.7 K 2024-11-28T06:55:34,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/cc55df6889634495b3df31ffea6392db as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/cc55df6889634495b3df31ffea6392db 2024-11-28T06:55:34,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/cc55df6889634495b3df31ffea6392db, entries=150, sequenceid=110, filesize=11.7 K 2024-11-28T06:55:34,334 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a23eb2f63cfcb5c9c85b72f46ebe560c in 1675ms, sequenceid=110, compaction requested=true 2024-11-28T06:55:34,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a23eb2f63cfcb5c9c85b72f46ebe560c: 2024-11-28T06:55:34,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a23eb2f63cfcb5c9c85b72f46ebe560c:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:55:34,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:34,334 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:55:34,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a23eb2f63cfcb5c9c85b72f46ebe560c:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:55:34,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:34,334 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:55:34,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a23eb2f63cfcb5c9c85b72f46ebe560c:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:55:34,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:55:34,335 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:55:34,335 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:55:34,335 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): a23eb2f63cfcb5c9c85b72f46ebe560c/B is initiating minor compaction (all files) 2024-11-28T06:55:34,335 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): a23eb2f63cfcb5c9c85b72f46ebe560c/A is initiating minor compaction (all files) 2024-11-28T06:55:34,335 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a23eb2f63cfcb5c9c85b72f46ebe560c/B in TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:34,335 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a23eb2f63cfcb5c9c85b72f46ebe560c/A in TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:34,335 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/b00c3b10e7f343e4bc8541d5d2671fb0, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/f556c8fe93824216bfa2dd923d7fc35b, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/cfed94df144d4d67ae259e63d864dcda] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp, totalSize=35.3 K 2024-11-28T06:55:34,335 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/631488c4571b4a18a7dd1c7db98ecea5, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/a1087292fa2b47e08a212d6aaacc0e98, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/a38e26179add4c799199dc8fb82b6979] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp, totalSize=99.2 K 2024-11-28T06:55:34,335 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:34,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-28T06:55:34,335 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. files: [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/631488c4571b4a18a7dd1c7db98ecea5, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/a1087292fa2b47e08a212d6aaacc0e98, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/a38e26179add4c799199dc8fb82b6979] 2024-11-28T06:55:34,336 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting b00c3b10e7f343e4bc8541d5d2671fb0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732776928478 2024-11-28T06:55:34,336 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 631488c4571b4a18a7dd1c7db98ecea5, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732776928478 2024-11-28T06:55:34,336 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1087292fa2b47e08a212d6aaacc0e98, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732776929756 2024-11-28T06:55:34,336 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting f556c8fe93824216bfa2dd923d7fc35b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732776929756 2024-11-28T06:55:34,336 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting a38e26179add4c799199dc8fb82b6979, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732776931950 2024-11-28T06:55:34,336 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting cfed94df144d4d67ae259e63d864dcda, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732776931950 2024-11-28T06:55:34,344 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a23eb2f63cfcb5c9c85b72f46ebe560c] 2024-11-28T06:55:34,345 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112847338d6f3e0e4f78be2e8ea168284a79_a23eb2f63cfcb5c9c85b72f46ebe560c store=[table=TestAcidGuarantees family=A region=a23eb2f63cfcb5c9c85b72f46ebe560c] 2024-11-28T06:55:34,346 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a23eb2f63cfcb5c9c85b72f46ebe560c#B#compaction#266 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:55:34,346 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/01ac587c37344178ba0247f8874c48cc is 50, key is test_row_0/B:col10/1732776932658/Put/seqid=0 2024-11-28T06:55:34,347 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112847338d6f3e0e4f78be2e8ea168284a79_a23eb2f63cfcb5c9c85b72f46ebe560c, store=[table=TestAcidGuarantees family=A region=a23eb2f63cfcb5c9c85b72f46ebe560c] 2024-11-28T06:55:34,347 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112847338d6f3e0e4f78be2e8ea168284a79_a23eb2f63cfcb5c9c85b72f46ebe560c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a23eb2f63cfcb5c9c85b72f46ebe560c] 2024-11-28T06:55:34,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742139_1315 (size=12207) 2024-11-28T06:55:34,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742140_1316 (size=4469) 2024-11-28T06:55:34,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-28T06:55:34,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:34,478 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing a23eb2f63cfcb5c9c85b72f46ebe560c 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T06:55:34,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=A 2024-11-28T06:55:34,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:34,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=B 2024-11-28T06:55:34,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:34,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=C 2024-11-28T06:55:34,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:34,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128e81daac444bd44759403494da490bc8c_a23eb2f63cfcb5c9c85b72f46ebe560c is 50, key is test_row_0/A:col10/1732776932770/Put/seqid=0 2024-11-28T06:55:34,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742141_1317 (size=12204) 2024-11-28T06:55:34,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-28T06:55:34,758 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a23eb2f63cfcb5c9c85b72f46ebe560c#A#compaction#265 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:55:34,758 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/ecbdca11b8004e32bcb6a37dec8cd187 is 175, key is test_row_0/A:col10/1732776932658/Put/seqid=0 2024-11-28T06:55:34,761 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/01ac587c37344178ba0247f8874c48cc as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/01ac587c37344178ba0247f8874c48cc 2024-11-28T06:55:34,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742142_1318 (size=31161) 2024-11-28T06:55:34,766 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a23eb2f63cfcb5c9c85b72f46ebe560c/B of a23eb2f63cfcb5c9c85b72f46ebe560c into 01ac587c37344178ba0247f8874c48cc(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:55:34,766 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/ecbdca11b8004e32bcb6a37dec8cd187 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/ecbdca11b8004e32bcb6a37dec8cd187 2024-11-28T06:55:34,766 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a23eb2f63cfcb5c9c85b72f46ebe560c: 2024-11-28T06:55:34,766 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., storeName=a23eb2f63cfcb5c9c85b72f46ebe560c/B, priority=13, startTime=1732776934334; duration=0sec 2024-11-28T06:55:34,766 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:55:34,766 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a23eb2f63cfcb5c9c85b72f46ebe560c:B 2024-11-28T06:55:34,766 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:55:34,767 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:55:34,767 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): a23eb2f63cfcb5c9c85b72f46ebe560c/C is initiating minor compaction (all files) 2024-11-28T06:55:34,767 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a23eb2f63cfcb5c9c85b72f46ebe560c/C in TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:34,767 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/1be44fd891304e3dab7129938f193889, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/44be8c6e67594af89b7f9216e130abd2, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/cc55df6889634495b3df31ffea6392db] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp, totalSize=35.3 K 2024-11-28T06:55:34,768 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 1be44fd891304e3dab7129938f193889, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732776928478 2024-11-28T06:55:34,768 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 44be8c6e67594af89b7f9216e130abd2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732776929756 2024-11-28T06:55:34,768 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting cc55df6889634495b3df31ffea6392db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732776931950 2024-11-28T06:55:34,770 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a23eb2f63cfcb5c9c85b72f46ebe560c/A of a23eb2f63cfcb5c9c85b72f46ebe560c into ecbdca11b8004e32bcb6a37dec8cd187(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:55:34,771 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a23eb2f63cfcb5c9c85b72f46ebe560c: 2024-11-28T06:55:34,771 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., storeName=a23eb2f63cfcb5c9c85b72f46ebe560c/A, priority=13, startTime=1732776934334; duration=0sec 2024-11-28T06:55:34,771 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:34,771 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a23eb2f63cfcb5c9c85b72f46ebe560c:A 2024-11-28T06:55:34,776 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a23eb2f63cfcb5c9c85b72f46ebe560c#C#compaction#268 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:55:34,777 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/25cc27fd9c054b7d8f9540efea100992 is 50, key is test_row_0/C:col10/1732776932658/Put/seqid=0 2024-11-28T06:55:34,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742143_1319 (size=12207) 2024-11-28T06:55:34,786 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/25cc27fd9c054b7d8f9540efea100992 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/25cc27fd9c054b7d8f9540efea100992 2024-11-28T06:55:34,793 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a23eb2f63cfcb5c9c85b72f46ebe560c/C of a23eb2f63cfcb5c9c85b72f46ebe560c into 25cc27fd9c054b7d8f9540efea100992(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:55:34,793 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a23eb2f63cfcb5c9c85b72f46ebe560c: 2024-11-28T06:55:34,793 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., storeName=a23eb2f63cfcb5c9c85b72f46ebe560c/C, priority=13, startTime=1732776934334; duration=0sec 2024-11-28T06:55:34,793 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:34,793 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a23eb2f63cfcb5c9c85b72f46ebe560c:C 2024-11-28T06:55:34,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:55:34,891 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128e81daac444bd44759403494da490bc8c_a23eb2f63cfcb5c9c85b72f46ebe560c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e81daac444bd44759403494da490bc8c_a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:34,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/3a19a250e1704ab4987960427be204be, store: [table=TestAcidGuarantees family=A region=a23eb2f63cfcb5c9c85b72f46ebe560c] 2024-11-28T06:55:34,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/3a19a250e1704ab4987960427be204be is 175, key is test_row_0/A:col10/1732776932770/Put/seqid=0 2024-11-28T06:55:34,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742144_1320 (size=31005) 2024-11-28T06:55:34,900 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/3a19a250e1704ab4987960427be204be 2024-11-28T06:55:34,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/bd2a76c4ae4f4615b14714428b615121 is 50, key is test_row_0/B:col10/1732776932770/Put/seqid=0 2024-11-28T06:55:34,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742145_1321 (size=12051) 2024-11-28T06:55:34,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:34,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. as already flushing 2024-11-28T06:55:35,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:35,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732776995007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:35,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:35,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:35,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:35,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:35,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732776995115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:35,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:35,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:35,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:35,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-28T06:55:35,168 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-28T06:55:35,168 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-28T06:55:35,314 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/bd2a76c4ae4f4615b14714428b615121 2024-11-28T06:55:35,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/a37a7cfb28b648e78d805405786b3022 is 50, key is test_row_0/C:col10/1732776932770/Put/seqid=0 2024-11-28T06:55:35,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742146_1322 (size=12051) 2024-11-28T06:55:35,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:35,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732776995325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:35,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:35,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:35,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:35,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:35,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732776995635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:35,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:35,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:35,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:35,775 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/a37a7cfb28b648e78d805405786b3022 2024-11-28T06:55:35,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/3a19a250e1704ab4987960427be204be as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/3a19a250e1704ab4987960427be204be 2024-11-28T06:55:35,781 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/3a19a250e1704ab4987960427be204be, entries=150, sequenceid=130, filesize=30.3 K 2024-11-28T06:55:35,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/bd2a76c4ae4f4615b14714428b615121 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/bd2a76c4ae4f4615b14714428b615121 2024-11-28T06:55:35,785 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/bd2a76c4ae4f4615b14714428b615121, entries=150, sequenceid=130, filesize=11.8 K 2024-11-28T06:55:35,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/a37a7cfb28b648e78d805405786b3022 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/a37a7cfb28b648e78d805405786b3022 2024-11-28T06:55:35,788 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/a37a7cfb28b648e78d805405786b3022, entries=150, sequenceid=130, filesize=11.8 K 2024-11-28T06:55:35,788 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a23eb2f63cfcb5c9c85b72f46ebe560c in 1311ms, sequenceid=130, compaction requested=false 2024-11-28T06:55:35,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for a23eb2f63cfcb5c9c85b72f46ebe560c: 2024-11-28T06:55:35,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:35,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-28T06:55:35,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-28T06:55:35,791 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-28T06:55:35,791 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7720 sec 2024-11-28T06:55:35,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 1.7760 sec 2024-11-28T06:55:36,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:36,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a23eb2f63cfcb5c9c85b72f46ebe560c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T06:55:36,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=A 2024-11-28T06:55:36,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:36,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=B 2024-11-28T06:55:36,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:36,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=C 2024-11-28T06:55:36,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:36,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112813e0ff9d038b486499daea6848ab2b22_a23eb2f63cfcb5c9c85b72f46ebe560c is 50, key is test_row_0/A:col10/1732776934998/Put/seqid=0 2024-11-28T06:55:36,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742147_1323 (size=14794) 2024-11-28T06:55:36,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-28T06:55:36,166 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:55:36,166 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:55:36,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-28T06:55:36,167 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:55:36,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-28T06:55:36,168 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:55:36,168 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:55:36,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:36,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732776996190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:36,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:36,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:36,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:36,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:36,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732776996195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:36,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:36,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:36,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:36,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-28T06:55:36,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:36,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732776996306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:36,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:36,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:36,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732776996306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:36,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:36,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:36,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:36,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:36,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:36,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-28T06:55:36,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:36,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. as already flushing 2024-11-28T06:55:36,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:36,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:36,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:36,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:36,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-28T06:55:36,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:36,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. as already flushing 2024-11-28T06:55:36,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:36,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:36,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:36,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-28T06:55:36,503 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:55:36,506 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112813e0ff9d038b486499daea6848ab2b22_a23eb2f63cfcb5c9c85b72f46ebe560c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112813e0ff9d038b486499daea6848ab2b22_a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:36,507 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/c338a105e94c4c07bd6bc0ec8faaeabc, store: [table=TestAcidGuarantees family=A region=a23eb2f63cfcb5c9c85b72f46ebe560c] 2024-11-28T06:55:36,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/c338a105e94c4c07bd6bc0ec8faaeabc is 175, key is test_row_0/A:col10/1732776934998/Put/seqid=0 2024-11-28T06:55:36,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742148_1324 (size=39749) 2024-11-28T06:55:36,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:36,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732776996515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:36,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:36,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732776996515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:36,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:36,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:36,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:36,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:36,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:36,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:36,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-28T06:55:36,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:36,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. as already flushing 2024-11-28T06:55:36,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:36,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:36,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:36,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:36,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-28T06:55:36,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:36,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. as already flushing 2024-11-28T06:55:36,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:36,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:36,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:36,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:36,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-28T06:55:36,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:36,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732776996825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:36,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:36,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732776996825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:36,911 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=146, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/c338a105e94c4c07bd6bc0ec8faaeabc 2024-11-28T06:55:36,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/832ddc81fec14e039b3279ca41eef084 is 50, key is test_row_0/B:col10/1732776934998/Put/seqid=0 2024-11-28T06:55:36,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742149_1325 (size=12151) 2024-11-28T06:55:36,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/832ddc81fec14e039b3279ca41eef084 2024-11-28T06:55:36,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/651d1981e0024c5fa8f4c2351be2b01a is 50, key is test_row_0/C:col10/1732776934998/Put/seqid=0 2024-11-28T06:55:36,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742150_1326 (size=12151) 2024-11-28T06:55:36,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-28T06:55:36,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:36,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. as already flushing 2024-11-28T06:55:36,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:36,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:36,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:36,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:36,932 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/651d1981e0024c5fa8f4c2351be2b01a 2024-11-28T06:55:36,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/c338a105e94c4c07bd6bc0ec8faaeabc as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/c338a105e94c4c07bd6bc0ec8faaeabc 2024-11-28T06:55:36,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/c338a105e94c4c07bd6bc0ec8faaeabc, entries=200, sequenceid=146, filesize=38.8 K 2024-11-28T06:55:36,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/832ddc81fec14e039b3279ca41eef084 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/832ddc81fec14e039b3279ca41eef084 2024-11-28T06:55:36,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/832ddc81fec14e039b3279ca41eef084, entries=150, sequenceid=146, filesize=11.9 K 2024-11-28T06:55:36,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/651d1981e0024c5fa8f4c2351be2b01a as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/651d1981e0024c5fa8f4c2351be2b01a 2024-11-28T06:55:36,947 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/651d1981e0024c5fa8f4c2351be2b01a, entries=150, sequenceid=146, filesize=11.9 K 2024-11-28T06:55:36,948 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for a23eb2f63cfcb5c9c85b72f46ebe560c in 853ms, sequenceid=146, compaction requested=true 2024-11-28T06:55:36,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a23eb2f63cfcb5c9c85b72f46ebe560c: 2024-11-28T06:55:36,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a23eb2f63cfcb5c9c85b72f46ebe560c:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:55:36,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:36,948 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:55:36,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a23eb2f63cfcb5c9c85b72f46ebe560c:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:55:36,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:36,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a23eb2f63cfcb5c9c85b72f46ebe560c:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:55:36,948 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:55:36,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:55:36,948 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101915 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:55:36,948 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36409 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:55:36,949 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): a23eb2f63cfcb5c9c85b72f46ebe560c/A is initiating minor compaction (all files) 2024-11-28T06:55:36,949 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): a23eb2f63cfcb5c9c85b72f46ebe560c/B is initiating minor compaction (all files) 2024-11-28T06:55:36,949 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a23eb2f63cfcb5c9c85b72f46ebe560c/A in TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:36,949 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a23eb2f63cfcb5c9c85b72f46ebe560c/B in TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:36,949 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/01ac587c37344178ba0247f8874c48cc, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/bd2a76c4ae4f4615b14714428b615121, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/832ddc81fec14e039b3279ca41eef084] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp, totalSize=35.6 K 2024-11-28T06:55:36,949 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/ecbdca11b8004e32bcb6a37dec8cd187, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/3a19a250e1704ab4987960427be204be, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/c338a105e94c4c07bd6bc0ec8faaeabc] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp, totalSize=99.5 K 2024-11-28T06:55:36,949 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:36,949 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. files: [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/ecbdca11b8004e32bcb6a37dec8cd187, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/3a19a250e1704ab4987960427be204be, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/c338a105e94c4c07bd6bc0ec8faaeabc] 2024-11-28T06:55:36,949 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 01ac587c37344178ba0247f8874c48cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732776931950 2024-11-28T06:55:36,949 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecbdca11b8004e32bcb6a37dec8cd187, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732776931950 2024-11-28T06:55:36,949 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting bd2a76c4ae4f4615b14714428b615121, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732776932770 2024-11-28T06:55:36,949 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a19a250e1704ab4987960427be204be, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732776932770 2024-11-28T06:55:36,949 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 832ddc81fec14e039b3279ca41eef084, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732776934998 2024-11-28T06:55:36,949 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting c338a105e94c4c07bd6bc0ec8faaeabc, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732776934994 2024-11-28T06:55:36,956 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a23eb2f63cfcb5c9c85b72f46ebe560c] 2024-11-28T06:55:36,957 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a23eb2f63cfcb5c9c85b72f46ebe560c#B#compaction#274 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:55:36,957 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112804c127f36ffb471db3f9f04e2f903ecc_a23eb2f63cfcb5c9c85b72f46ebe560c store=[table=TestAcidGuarantees family=A region=a23eb2f63cfcb5c9c85b72f46ebe560c] 2024-11-28T06:55:36,957 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/99993b8b93ca47478de4ee5bbdc622c5 is 50, key is test_row_0/B:col10/1732776934998/Put/seqid=0 2024-11-28T06:55:36,960 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112804c127f36ffb471db3f9f04e2f903ecc_a23eb2f63cfcb5c9c85b72f46ebe560c, store=[table=TestAcidGuarantees family=A region=a23eb2f63cfcb5c9c85b72f46ebe560c] 2024-11-28T06:55:36,960 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112804c127f36ffb471db3f9f04e2f903ecc_a23eb2f63cfcb5c9c85b72f46ebe560c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a23eb2f63cfcb5c9c85b72f46ebe560c] 2024-11-28T06:55:36,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742151_1327 (size=12459) 2024-11-28T06:55:36,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742152_1328 (size=4469) 2024-11-28T06:55:36,975 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/99993b8b93ca47478de4ee5bbdc622c5 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/99993b8b93ca47478de4ee5bbdc622c5 2024-11-28T06:55:36,979 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a23eb2f63cfcb5c9c85b72f46ebe560c/B of a23eb2f63cfcb5c9c85b72f46ebe560c into 99993b8b93ca47478de4ee5bbdc622c5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:55:36,979 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a23eb2f63cfcb5c9c85b72f46ebe560c: 2024-11-28T06:55:36,980 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., storeName=a23eb2f63cfcb5c9c85b72f46ebe560c/B, priority=13, startTime=1732776936948; duration=0sec 2024-11-28T06:55:36,980 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:55:36,980 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a23eb2f63cfcb5c9c85b72f46ebe560c:B 2024-11-28T06:55:36,980 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:55:36,981 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36409 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:55:36,981 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): a23eb2f63cfcb5c9c85b72f46ebe560c/C is initiating minor compaction (all files) 2024-11-28T06:55:36,981 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a23eb2f63cfcb5c9c85b72f46ebe560c/C in TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:36,981 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/25cc27fd9c054b7d8f9540efea100992, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/a37a7cfb28b648e78d805405786b3022, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/651d1981e0024c5fa8f4c2351be2b01a] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp, totalSize=35.6 K 2024-11-28T06:55:36,981 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 25cc27fd9c054b7d8f9540efea100992, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732776931950 2024-11-28T06:55:36,981 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting a37a7cfb28b648e78d805405786b3022, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732776932770 2024-11-28T06:55:36,981 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 651d1981e0024c5fa8f4c2351be2b01a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732776934998 2024-11-28T06:55:36,987 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a23eb2f63cfcb5c9c85b72f46ebe560c#C#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:55:36,987 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/29523127218e41e28b5782ca8a6240b8 is 50, key is test_row_0/C:col10/1732776934998/Put/seqid=0 2024-11-28T06:55:36,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742153_1329 (size=12459) 2024-11-28T06:55:37,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-28T06:55:37,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:37,084 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing a23eb2f63cfcb5c9c85b72f46ebe560c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T06:55:37,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=A 2024-11-28T06:55:37,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:37,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=B 2024-11-28T06:55:37,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:37,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=C 2024-11-28T06:55:37,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:37,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411280f66a4621b35450fb8d8e8a4ae6deb3c_a23eb2f63cfcb5c9c85b72f46ebe560c is 50, key is test_row_0/A:col10/1732776936181/Put/seqid=0 2024-11-28T06:55:37,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742154_1330 (size=12304) 2024-11-28T06:55:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-28T06:55:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:37,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. as already flushing 2024-11-28T06:55:37,373 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a23eb2f63cfcb5c9c85b72f46ebe560c#A#compaction#275 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:55:37,373 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/a9c1b4a5b7cb469e917c3ee92694e497 is 175, key is test_row_0/A:col10/1732776934998/Put/seqid=0 2024-11-28T06:55:37,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742155_1331 (size=31413) 2024-11-28T06:55:37,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732776997366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:37,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:37,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:37,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:37,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732776997367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:37,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:37,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:37,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:37,396 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/29523127218e41e28b5782ca8a6240b8 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/29523127218e41e28b5782ca8a6240b8 2024-11-28T06:55:37,400 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a23eb2f63cfcb5c9c85b72f46ebe560c/C of a23eb2f63cfcb5c9c85b72f46ebe560c into 29523127218e41e28b5782ca8a6240b8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:55:37,400 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a23eb2f63cfcb5c9c85b72f46ebe560c: 2024-11-28T06:55:37,400 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., storeName=a23eb2f63cfcb5c9c85b72f46ebe560c/C, priority=13, startTime=1732776936948; duration=0sec 2024-11-28T06:55:37,400 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:37,400 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a23eb2f63cfcb5c9c85b72f46ebe560c:C 2024-11-28T06:55:37,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:37,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732776997486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:37,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:37,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:37,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:37,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:37,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732776997486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:37,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:37,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:37,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:37,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:55:37,508 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411280f66a4621b35450fb8d8e8a4ae6deb3c_a23eb2f63cfcb5c9c85b72f46ebe560c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411280f66a4621b35450fb8d8e8a4ae6deb3c_a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:37,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/cc03d86d010e4ea2aa958c82e26cb252, store: [table=TestAcidGuarantees family=A region=a23eb2f63cfcb5c9c85b72f46ebe560c] 2024-11-28T06:55:37,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/cc03d86d010e4ea2aa958c82e26cb252 is 175, key is test_row_0/A:col10/1732776936181/Put/seqid=0 2024-11-28T06:55:37,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742156_1332 (size=31105) 2024-11-28T06:55:37,515 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=166, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/cc03d86d010e4ea2aa958c82e26cb252 2024-11-28T06:55:37,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/178c80765bd7483ea85cdb04addd790c is 50, key is test_row_0/B:col10/1732776936181/Put/seqid=0 2024-11-28T06:55:37,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742157_1333 (size=12151) 2024-11-28T06:55:37,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:37,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732776997705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:37,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:37,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:37,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732776997705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:37,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:37,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:37,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:37,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:37,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:37,781 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/a9c1b4a5b7cb469e917c3ee92694e497 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/a9c1b4a5b7cb469e917c3ee92694e497 2024-11-28T06:55:37,786 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a23eb2f63cfcb5c9c85b72f46ebe560c/A of a23eb2f63cfcb5c9c85b72f46ebe560c into a9c1b4a5b7cb469e917c3ee92694e497(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:55:37,786 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a23eb2f63cfcb5c9c85b72f46ebe560c: 2024-11-28T06:55:37,786 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., storeName=a23eb2f63cfcb5c9c85b72f46ebe560c/A, priority=13, startTime=1732776936948; duration=0sec 2024-11-28T06:55:37,786 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:37,786 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a23eb2f63cfcb5c9c85b72f46ebe560c:A 2024-11-28T06:55:37,925 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/178c80765bd7483ea85cdb04addd790c 2024-11-28T06:55:37,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/9dbdd395d2c34eb1ba247783b2e7ff63 is 50, key is test_row_0/C:col10/1732776936181/Put/seqid=0 2024-11-28T06:55:37,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742158_1334 (size=12151) 2024-11-28T06:55:38,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:38,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732776998026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:38,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:38,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:38,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:38,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:38,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732776998026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:38,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:38,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:38,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:38,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:38,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1732776998075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:38,081 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_2' in a23eb2f63cfcb5c9c85b72f46ebe560c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8288 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:55:38,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:38,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:38,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:38,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:38,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40022 deadline: 1732776998086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:38,092 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_2' in a23eb2f63cfcb5c9c85b72f46ebe560c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8283 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:55:38,093 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:38,093 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:38,093 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:38,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:38,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40084 deadline: 1732776998095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:38,104 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_1' in a23eb2f63cfcb5c9c85b72f46ebe560c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8296 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:55:38,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:38,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:38,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:38,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-28T06:55:38,340 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/9dbdd395d2c34eb1ba247783b2e7ff63 2024-11-28T06:55:38,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/cc03d86d010e4ea2aa958c82e26cb252 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/cc03d86d010e4ea2aa958c82e26cb252 2024-11-28T06:55:38,347 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/cc03d86d010e4ea2aa958c82e26cb252, entries=150, sequenceid=166, filesize=30.4 K 2024-11-28T06:55:38,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/178c80765bd7483ea85cdb04addd790c as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/178c80765bd7483ea85cdb04addd790c 2024-11-28T06:55:38,351 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/178c80765bd7483ea85cdb04addd790c, entries=150, sequenceid=166, filesize=11.9 K 2024-11-28T06:55:38,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/9dbdd395d2c34eb1ba247783b2e7ff63 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/9dbdd395d2c34eb1ba247783b2e7ff63 2024-11-28T06:55:38,355 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/9dbdd395d2c34eb1ba247783b2e7ff63, entries=150, sequenceid=166, filesize=11.9 K 2024-11-28T06:55:38,356 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for a23eb2f63cfcb5c9c85b72f46ebe560c in 1271ms, sequenceid=166, compaction requested=false 2024-11-28T06:55:38,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for a23eb2f63cfcb5c9c85b72f46ebe560c: 2024-11-28T06:55:38,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:38,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-28T06:55:38,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-28T06:55:38,358 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-28T06:55:38,358 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1890 sec 2024-11-28T06:55:38,359 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 2.1920 sec 2024-11-28T06:55:38,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:38,550 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a23eb2f63cfcb5c9c85b72f46ebe560c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T06:55:38,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=A 2024-11-28T06:55:38,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:38,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=B 2024-11-28T06:55:38,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:38,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=C 2024-11-28T06:55:38,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:38,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128d9ac9db4aee64e2b9f9635bcc9e975e5_a23eb2f63cfcb5c9c85b72f46ebe560c is 50, key is test_row_0/A:col10/1732776938550/Put/seqid=0 2024-11-28T06:55:38,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742159_1335 (size=17284) 2024-11-28T06:55:38,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:38,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732776998599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:38,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:38,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:38,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:38,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:38,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732776998604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:38,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:38,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:38,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:38,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732776998715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:38,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:38,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732776998715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:38,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:38,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:38,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:38,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:38,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:38,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:38,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732776998925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:38,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:38,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:38,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:38,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:38,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732776998925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:38,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:38,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:38,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:38,964 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:55:38,967 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128d9ac9db4aee64e2b9f9635bcc9e975e5_a23eb2f63cfcb5c9c85b72f46ebe560c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d9ac9db4aee64e2b9f9635bcc9e975e5_a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:38,968 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/e458197de130470cad31676ad877c68f, store: [table=TestAcidGuarantees family=A region=a23eb2f63cfcb5c9c85b72f46ebe560c] 2024-11-28T06:55:38,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/e458197de130470cad31676ad877c68f is 175, key is test_row_0/A:col10/1732776938550/Put/seqid=0 2024-11-28T06:55:38,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742160_1336 (size=48389) 2024-11-28T06:55:39,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:39,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732776999246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:39,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:39,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:39,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:39,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:39,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732776999246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:39,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:39,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:39,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:39,373 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=183, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/e458197de130470cad31676ad877c68f 2024-11-28T06:55:39,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/01f837ec6b9c475ea21d817324f85848 is 50, key is test_row_0/B:col10/1732776938550/Put/seqid=0 2024-11-28T06:55:39,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742161_1337 (size=12151) 2024-11-28T06:55:39,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:39,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732776999765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:39,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:39,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:39,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:39,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732776999765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:39,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:39,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:39,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:39,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:39,789 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/01f837ec6b9c475ea21d817324f85848 2024-11-28T06:55:39,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/386e78186621453d990711fc8fd75c58 is 50, key is test_row_0/C:col10/1732776938550/Put/seqid=0 2024-11-28T06:55:39,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742162_1338 (size=12151) 2024-11-28T06:55:40,199 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/386e78186621453d990711fc8fd75c58 2024-11-28T06:55:40,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/e458197de130470cad31676ad877c68f as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/e458197de130470cad31676ad877c68f 2024-11-28T06:55:40,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/e458197de130470cad31676ad877c68f, entries=250, sequenceid=183, filesize=47.3 K 2024-11-28T06:55:40,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/01f837ec6b9c475ea21d817324f85848 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/01f837ec6b9c475ea21d817324f85848 2024-11-28T06:55:40,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/01f837ec6b9c475ea21d817324f85848, entries=150, sequenceid=183, filesize=11.9 K 2024-11-28T06:55:40,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/386e78186621453d990711fc8fd75c58 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/386e78186621453d990711fc8fd75c58 2024-11-28T06:55:40,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/386e78186621453d990711fc8fd75c58, entries=150, sequenceid=183, filesize=11.9 K 2024-11-28T06:55:40,224 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for a23eb2f63cfcb5c9c85b72f46ebe560c in 1674ms, sequenceid=183, compaction requested=true 2024-11-28T06:55:40,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a23eb2f63cfcb5c9c85b72f46ebe560c: 2024-11-28T06:55:40,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a23eb2f63cfcb5c9c85b72f46ebe560c:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:55:40,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:40,224 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:55:40,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a23eb2f63cfcb5c9c85b72f46ebe560c:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:55:40,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:40,224 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:55:40,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a23eb2f63cfcb5c9c85b72f46ebe560c:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:55:40,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:55:40,226 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:55:40,226 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110907 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:55:40,226 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): a23eb2f63cfcb5c9c85b72f46ebe560c/A is initiating minor compaction (all files) 2024-11-28T06:55:40,226 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): a23eb2f63cfcb5c9c85b72f46ebe560c/B is initiating minor compaction (all files) 2024-11-28T06:55:40,226 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a23eb2f63cfcb5c9c85b72f46ebe560c/B in TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:40,226 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a23eb2f63cfcb5c9c85b72f46ebe560c/A in TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:40,226 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/99993b8b93ca47478de4ee5bbdc622c5, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/178c80765bd7483ea85cdb04addd790c, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/01f837ec6b9c475ea21d817324f85848] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp, totalSize=35.9 K 2024-11-28T06:55:40,226 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/a9c1b4a5b7cb469e917c3ee92694e497, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/cc03d86d010e4ea2aa958c82e26cb252, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/e458197de130470cad31676ad877c68f] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp, totalSize=108.3 K 2024-11-28T06:55:40,226 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:40,226 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. files: [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/a9c1b4a5b7cb469e917c3ee92694e497, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/cc03d86d010e4ea2aa958c82e26cb252, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/e458197de130470cad31676ad877c68f] 2024-11-28T06:55:40,226 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 99993b8b93ca47478de4ee5bbdc622c5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732776934998 2024-11-28T06:55:40,226 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9c1b4a5b7cb469e917c3ee92694e497, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732776934998 2024-11-28T06:55:40,227 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 178c80765bd7483ea85cdb04addd790c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732776936181 2024-11-28T06:55:40,227 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc03d86d010e4ea2aa958c82e26cb252, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732776936181 2024-11-28T06:55:40,227 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 01f837ec6b9c475ea21d817324f85848, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732776937343 2024-11-28T06:55:40,227 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting e458197de130470cad31676ad877c68f, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732776937343 2024-11-28T06:55:40,232 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a23eb2f63cfcb5c9c85b72f46ebe560c] 2024-11-28T06:55:40,240 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112854896e5d93084059bd422fc407def583_a23eb2f63cfcb5c9c85b72f46ebe560c store=[table=TestAcidGuarantees family=A region=a23eb2f63cfcb5c9c85b72f46ebe560c] 2024-11-28T06:55:40,242 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a23eb2f63cfcb5c9c85b72f46ebe560c#B#compaction#284 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:55:40,242 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112854896e5d93084059bd422fc407def583_a23eb2f63cfcb5c9c85b72f46ebe560c, store=[table=TestAcidGuarantees family=A region=a23eb2f63cfcb5c9c85b72f46ebe560c] 2024-11-28T06:55:40,242 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/76f8ff50c5294e608fbaa00161200402 is 50, key is test_row_0/B:col10/1732776938550/Put/seqid=0 2024-11-28T06:55:40,243 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112854896e5d93084059bd422fc407def583_a23eb2f63cfcb5c9c85b72f46ebe560c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a23eb2f63cfcb5c9c85b72f46ebe560c] 2024-11-28T06:55:40,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742163_1339 (size=12561) 2024-11-28T06:55:40,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742164_1340 (size=4469) 2024-11-28T06:55:40,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-28T06:55:40,345 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:55:40,346 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:55:40,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-28T06:55:40,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-28T06:55:40,347 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:55:40,348 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:55:40,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:55:40,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-28T06:55:40,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-28T06:55:40,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:40,500 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing a23eb2f63cfcb5c9c85b72f46ebe560c 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-28T06:55:40,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=A 2024-11-28T06:55:40,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:40,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=B 2024-11-28T06:55:40,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:40,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=C 2024-11-28T06:55:40,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:40,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411288851490c0d03434cae97ab4c7710aec7_a23eb2f63cfcb5c9c85b72f46ebe560c is 50, key is test_row_0/A:col10/1732776938597/Put/seqid=0 2024-11-28T06:55:40,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742165_1341 (size=12304) 2024-11-28T06:55:40,651 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a23eb2f63cfcb5c9c85b72f46ebe560c#A#compaction#283 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:55:40,651 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/1ad12571d0c1433084422901f244bb0a is 175, key is test_row_0/A:col10/1732776938550/Put/seqid=0 2024-11-28T06:55:40,654 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/76f8ff50c5294e608fbaa00161200402 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/76f8ff50c5294e608fbaa00161200402 2024-11-28T06:55:40,658 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a23eb2f63cfcb5c9c85b72f46ebe560c/B of a23eb2f63cfcb5c9c85b72f46ebe560c into 76f8ff50c5294e608fbaa00161200402(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:55:40,658 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a23eb2f63cfcb5c9c85b72f46ebe560c: 2024-11-28T06:55:40,658 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., storeName=a23eb2f63cfcb5c9c85b72f46ebe560c/B, priority=13, startTime=1732776940224; duration=0sec 2024-11-28T06:55:40,658 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:55:40,658 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a23eb2f63cfcb5c9c85b72f46ebe560c:B 2024-11-28T06:55:40,659 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:55:40,659 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:55:40,660 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): a23eb2f63cfcb5c9c85b72f46ebe560c/C is initiating minor compaction (all files) 2024-11-28T06:55:40,660 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a23eb2f63cfcb5c9c85b72f46ebe560c/C in TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:40,660 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/29523127218e41e28b5782ca8a6240b8, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/9dbdd395d2c34eb1ba247783b2e7ff63, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/386e78186621453d990711fc8fd75c58] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp, totalSize=35.9 K 2024-11-28T06:55:40,660 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 29523127218e41e28b5782ca8a6240b8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732776934998 2024-11-28T06:55:40,660 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dbdd395d2c34eb1ba247783b2e7ff63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732776936181 2024-11-28T06:55:40,661 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 386e78186621453d990711fc8fd75c58, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732776937343 2024-11-28T06:55:40,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-28T06:55:40,668 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a23eb2f63cfcb5c9c85b72f46ebe560c#C#compaction#286 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:55:40,668 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/26cdd831f5714ac2a9bc413d5fe48543 is 50, key is test_row_0/C:col10/1732776938550/Put/seqid=0 2024-11-28T06:55:40,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742166_1342 (size=31515) 2024-11-28T06:55:40,686 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/1ad12571d0c1433084422901f244bb0a as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/1ad12571d0c1433084422901f244bb0a 2024-11-28T06:55:40,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742167_1343 (size=12561) 2024-11-28T06:55:40,691 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a23eb2f63cfcb5c9c85b72f46ebe560c/A of a23eb2f63cfcb5c9c85b72f46ebe560c into 1ad12571d0c1433084422901f244bb0a(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:55:40,691 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a23eb2f63cfcb5c9c85b72f46ebe560c: 2024-11-28T06:55:40,691 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., storeName=a23eb2f63cfcb5c9c85b72f46ebe560c/A, priority=13, startTime=1732776940224; duration=0sec 2024-11-28T06:55:40,691 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:40,691 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a23eb2f63cfcb5c9c85b72f46ebe560c:A 2024-11-28T06:55:40,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:40,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. as already flushing 2024-11-28T06:55:40,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:40,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732777000821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:40,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:40,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:40,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:40,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:40,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732777000824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:40,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:40,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:40,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:40,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:55:40,913 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411288851490c0d03434cae97ab4c7710aec7_a23eb2f63cfcb5c9c85b72f46ebe560c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288851490c0d03434cae97ab4c7710aec7_a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:40,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/1d10f70d03b8492882922f730b75029e, store: [table=TestAcidGuarantees family=A region=a23eb2f63cfcb5c9c85b72f46ebe560c] 2024-11-28T06:55:40,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/1d10f70d03b8492882922f730b75029e is 175, key is test_row_0/A:col10/1732776938597/Put/seqid=0 2024-11-28T06:55:40,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742168_1344 (size=31105) 2024-11-28T06:55:40,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:40,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732777000935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:40,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:40,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:40,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:40,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:40,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732777000935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:40,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-28T06:55:41,093 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/26cdd831f5714ac2a9bc413d5fe48543 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/26cdd831f5714ac2a9bc413d5fe48543 2024-11-28T06:55:41,096 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a23eb2f63cfcb5c9c85b72f46ebe560c/C of a23eb2f63cfcb5c9c85b72f46ebe560c into 26cdd831f5714ac2a9bc413d5fe48543(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:55:41,096 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a23eb2f63cfcb5c9c85b72f46ebe560c: 2024-11-28T06:55:41,096 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., storeName=a23eb2f63cfcb5c9c85b72f46ebe560c/C, priority=13, startTime=1732776940224; duration=0sec 2024-11-28T06:55:41,097 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:41,097 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a23eb2f63cfcb5c9c85b72f46ebe560c:C 2024-11-28T06:55:41,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:41,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732777001145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:41,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:41,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:41,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:41,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:41,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732777001155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:41,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:41,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:41,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:41,318 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=200, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/1d10f70d03b8492882922f730b75029e 2024-11-28T06:55:41,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/a78f1d6bcc83401aabd56a47ec82c0ee is 50, key is test_row_0/B:col10/1732776938597/Put/seqid=0 2024-11-28T06:55:41,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742169_1345 (size=12151) 2024-11-28T06:55:41,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:41,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732777001456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:41,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:41,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:41,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:41,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:41,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732777001465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:41,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:41,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:41,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:41,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-28T06:55:41,728 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/a78f1d6bcc83401aabd56a47ec82c0ee 2024-11-28T06:55:41,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/8268f0a32d154965b5249f5fc63a24ad is 50, key is test_row_0/C:col10/1732776938597/Put/seqid=0 2024-11-28T06:55:41,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742170_1346 (size=12151) 2024-11-28T06:55:41,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:41,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732777001966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:41,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:41,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:41,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:41,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732777001975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:41,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:41,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:41,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:42,137 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/8268f0a32d154965b5249f5fc63a24ad 2024-11-28T06:55:42,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/1d10f70d03b8492882922f730b75029e as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/1d10f70d03b8492882922f730b75029e 2024-11-28T06:55:42,144 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/1d10f70d03b8492882922f730b75029e, entries=150, sequenceid=200, filesize=30.4 K 2024-11-28T06:55:42,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/a78f1d6bcc83401aabd56a47ec82c0ee as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/a78f1d6bcc83401aabd56a47ec82c0ee 2024-11-28T06:55:42,147 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/a78f1d6bcc83401aabd56a47ec82c0ee, entries=150, sequenceid=200, filesize=11.9 K 2024-11-28T06:55:42,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/8268f0a32d154965b5249f5fc63a24ad as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/8268f0a32d154965b5249f5fc63a24ad 2024-11-28T06:55:42,151 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/8268f0a32d154965b5249f5fc63a24ad, entries=150, sequenceid=200, filesize=11.9 K 2024-11-28T06:55:42,152 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for a23eb2f63cfcb5c9c85b72f46ebe560c in 1652ms, sequenceid=200, compaction requested=false 2024-11-28T06:55:42,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for a23eb2f63cfcb5c9c85b72f46ebe560c: 2024-11-28T06:55:42,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:42,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-28T06:55:42,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-28T06:55:42,155 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-28T06:55:42,155 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8060 sec 2024-11-28T06:55:42,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.8090 sec 2024-11-28T06:55:42,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-28T06:55:42,496 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:55:42,497 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:55:42,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-28T06:55:42,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-28T06:55:42,498 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:55:42,499 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:55:42,499 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:55:42,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-28T06:55:42,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-28T06:55:42,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:42,651 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing a23eb2f63cfcb5c9c85b72f46ebe560c 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-28T06:55:42,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=A 2024-11-28T06:55:42,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:42,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=B 2024-11-28T06:55:42,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:42,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=C 2024-11-28T06:55:42,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:42,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411285d131360e2a249488fa9471f4f81974a_a23eb2f63cfcb5c9c85b72f46ebe560c is 50, key is test_row_0/A:col10/1732776940823/Put/seqid=0 2024-11-28T06:55:42,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742171_1347 (size=12304) 2024-11-28T06:55:42,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-28T06:55:42,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:42,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. as already flushing 2024-11-28T06:55:43,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:43,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732777003044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:43,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:43,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:43,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:43,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:43,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732777003045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:43,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:43,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:43,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:43,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:55:43,065 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411285d131360e2a249488fa9471f4f81974a_a23eb2f63cfcb5c9c85b72f46ebe560c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411285d131360e2a249488fa9471f4f81974a_a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:43,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/905117eab9b444bb867425c2e97958ed, store: [table=TestAcidGuarantees family=A region=a23eb2f63cfcb5c9c85b72f46ebe560c] 2024-11-28T06:55:43,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/905117eab9b444bb867425c2e97958ed is 175, key is test_row_0/A:col10/1732776940823/Put/seqid=0 2024-11-28T06:55:43,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742172_1348 (size=31105) 2024-11-28T06:55:43,072 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=218, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/905117eab9b444bb867425c2e97958ed 2024-11-28T06:55:43,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/7b56fb8e89504bbba6e5f3be3d28aff3 is 50, key is test_row_0/B:col10/1732776940823/Put/seqid=0 2024-11-28T06:55:43,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742173_1349 (size=12151) 2024-11-28T06:55:43,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-28T06:55:43,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:43,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732777003155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:43,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:43,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:43,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:43,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:43,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732777003166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:43,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:43,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:43,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:43,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:43,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732777003365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:43,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:43,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:43,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:43,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:43,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732777003375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:43,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:43,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:43,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:43,485 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/7b56fb8e89504bbba6e5f3be3d28aff3 2024-11-28T06:55:43,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/bc249dfb6ca24e84827e524e83604133 is 50, key is test_row_0/C:col10/1732776940823/Put/seqid=0 2024-11-28T06:55:43,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742174_1350 (size=12151) 2024-11-28T06:55:43,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-28T06:55:43,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:43,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732777003676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:43,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:43,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:43,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:43,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:43,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732777003685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:43,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:43,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:43,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:43,869 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T06:55:43,895 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/bc249dfb6ca24e84827e524e83604133 2024-11-28T06:55:43,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/905117eab9b444bb867425c2e97958ed as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/905117eab9b444bb867425c2e97958ed 2024-11-28T06:55:43,902 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/905117eab9b444bb867425c2e97958ed, entries=150, sequenceid=218, filesize=30.4 K 2024-11-28T06:55:43,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/7b56fb8e89504bbba6e5f3be3d28aff3 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/7b56fb8e89504bbba6e5f3be3d28aff3 2024-11-28T06:55:43,906 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/7b56fb8e89504bbba6e5f3be3d28aff3, entries=150, sequenceid=218, filesize=11.9 K 2024-11-28T06:55:43,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/bc249dfb6ca24e84827e524e83604133 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/bc249dfb6ca24e84827e524e83604133 2024-11-28T06:55:43,909 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/bc249dfb6ca24e84827e524e83604133, entries=150, sequenceid=218, filesize=11.9 K 2024-11-28T06:55:43,910 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for a23eb2f63cfcb5c9c85b72f46ebe560c in 1259ms, sequenceid=218, compaction requested=true 2024-11-28T06:55:43,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for a23eb2f63cfcb5c9c85b72f46ebe560c: 2024-11-28T06:55:43,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:43,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-28T06:55:43,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-28T06:55:43,913 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-28T06:55:43,913 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4120 sec 2024-11-28T06:55:43,916 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.4160 sec 2024-11-28T06:55:44,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:44,202 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a23eb2f63cfcb5c9c85b72f46ebe560c 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T06:55:44,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=A 2024-11-28T06:55:44,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:44,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=B 2024-11-28T06:55:44,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:44,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=C 2024-11-28T06:55:44,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:44,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128f55cc711c61c4f22b5d224a3c1e1b48b_a23eb2f63cfcb5c9c85b72f46ebe560c is 50, key is test_row_0/A:col10/1732776944201/Put/seqid=0 2024-11-28T06:55:44,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742175_1351 (size=14794) 2024-11-28T06:55:44,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:44,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732777004237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:44,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:44,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732777004241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:44,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:44,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:44,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:44,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:44,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:44,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:44,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:44,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732777004355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:44,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:44,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:44,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732777004355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:44,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:44,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:44,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:44,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:44,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:44,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:44,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732777004576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:44,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:44,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732777004576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:44,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:44,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:44,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:44,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:44,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:44,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:44,612 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:55:44,615 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128f55cc711c61c4f22b5d224a3c1e1b48b_a23eb2f63cfcb5c9c85b72f46ebe560c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128f55cc711c61c4f22b5d224a3c1e1b48b_a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:44,616 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/25b07c176652456fa6f1e8dd3fce48cc, store: [table=TestAcidGuarantees family=A region=a23eb2f63cfcb5c9c85b72f46ebe560c] 2024-11-28T06:55:44,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/25b07c176652456fa6f1e8dd3fce48cc is 175, key is test_row_0/A:col10/1732776944201/Put/seqid=0 2024-11-28T06:55:44,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742176_1352 (size=39749) 2024-11-28T06:55:44,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-28T06:55:44,646 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:55:44,646 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:55:44,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-28T06:55:44,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-28T06:55:44,648 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:55:44,649 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:55:44,649 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:55:44,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-28T06:55:44,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-28T06:55:44,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:44,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. as already flushing 2024-11-28T06:55:44,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:44,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:44,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:44,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:44,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:44,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732777004895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:44,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:44,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:44,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732777004896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:44,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:44,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:44,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:44,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:44,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:44,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-28T06:55:44,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:44,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. as already flushing 2024-11-28T06:55:44,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:44,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:44,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:44,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:44,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-28T06:55:45,019 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=236, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/25b07c176652456fa6f1e8dd3fce48cc 2024-11-28T06:55:45,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/4c22987c0f814dd9a651e61112f03805 is 50, key is test_row_0/B:col10/1732776944201/Put/seqid=0 2024-11-28T06:55:45,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742177_1353 (size=12151) 2024-11-28T06:55:45,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-28T06:55:45,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:45,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. as already flushing 2024-11-28T06:55:45,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:45,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:45,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:45,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:45,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-28T06:55:45,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:45,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. as already flushing 2024-11-28T06:55:45,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:45,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:45,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:45,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:45,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-28T06:55:45,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-28T06:55:45,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:45,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. as already flushing 2024-11-28T06:55:45,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:45,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:45,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:45,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:45,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:45,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732777005416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:45,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:45,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:45,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732777005416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:45,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:45,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:45,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:45,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:45,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:45,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/4c22987c0f814dd9a651e61112f03805 2024-11-28T06:55:45,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/6baa231a50474600bc96fa4244f0c727 is 50, key is test_row_0/C:col10/1732776944201/Put/seqid=0 2024-11-28T06:55:45,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742178_1354 (size=12151) 2024-11-28T06:55:45,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-28T06:55:45,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:45,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. as already flushing 2024-11-28T06:55:45,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:45,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:45,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:45,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:45,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-28T06:55:45,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:45,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. as already flushing 2024-11-28T06:55:45,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:45,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:45,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:45,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:45,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-28T06:55:45,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/6baa231a50474600bc96fa4244f0c727 2024-11-28T06:55:45,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-28T06:55:45,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:45,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. as already flushing 2024-11-28T06:55:45,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:45,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:45,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:45,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:45,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/25b07c176652456fa6f1e8dd3fce48cc as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/25b07c176652456fa6f1e8dd3fce48cc 2024-11-28T06:55:45,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/25b07c176652456fa6f1e8dd3fce48cc, entries=200, sequenceid=236, filesize=38.8 K 2024-11-28T06:55:45,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/4c22987c0f814dd9a651e61112f03805 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/4c22987c0f814dd9a651e61112f03805 2024-11-28T06:55:45,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/4c22987c0f814dd9a651e61112f03805, entries=150, sequenceid=236, filesize=11.9 K 2024-11-28T06:55:45,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/6baa231a50474600bc96fa4244f0c727 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/6baa231a50474600bc96fa4244f0c727 2024-11-28T06:55:45,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/6baa231a50474600bc96fa4244f0c727, entries=150, sequenceid=236, filesize=11.9 K 2024-11-28T06:55:45,894 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for a23eb2f63cfcb5c9c85b72f46ebe560c in 1692ms, sequenceid=236, compaction requested=true 2024-11-28T06:55:45,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a23eb2f63cfcb5c9c85b72f46ebe560c: 2024-11-28T06:55:45,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a23eb2f63cfcb5c9c85b72f46ebe560c:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:55:45,894 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T06:55:45,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:45,894 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T06:55:45,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a23eb2f63cfcb5c9c85b72f46ebe560c:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:55:45,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:45,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a23eb2f63cfcb5c9c85b72f46ebe560c:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:55:45,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:55:45,896 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133474 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T06:55:45,896 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T06:55:45,896 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): a23eb2f63cfcb5c9c85b72f46ebe560c/B is initiating minor compaction (all files) 2024-11-28T06:55:45,896 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): a23eb2f63cfcb5c9c85b72f46ebe560c/A is initiating minor compaction (all files) 2024-11-28T06:55:45,896 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a23eb2f63cfcb5c9c85b72f46ebe560c/B in TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:45,896 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a23eb2f63cfcb5c9c85b72f46ebe560c/A in TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:45,896 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/76f8ff50c5294e608fbaa00161200402, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/a78f1d6bcc83401aabd56a47ec82c0ee, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/7b56fb8e89504bbba6e5f3be3d28aff3, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/4c22987c0f814dd9a651e61112f03805] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp, totalSize=47.9 K 2024-11-28T06:55:45,896 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/1ad12571d0c1433084422901f244bb0a, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/1d10f70d03b8492882922f730b75029e, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/905117eab9b444bb867425c2e97958ed, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/25b07c176652456fa6f1e8dd3fce48cc] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp, totalSize=130.3 K 2024-11-28T06:55:45,896 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:45,896 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. files: [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/1ad12571d0c1433084422901f244bb0a, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/1d10f70d03b8492882922f730b75029e, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/905117eab9b444bb867425c2e97958ed, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/25b07c176652456fa6f1e8dd3fce48cc] 2024-11-28T06:55:45,896 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 76f8ff50c5294e608fbaa00161200402, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732776937343 2024-11-28T06:55:45,896 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ad12571d0c1433084422901f244bb0a, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732776937343 2024-11-28T06:55:45,896 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting a78f1d6bcc83401aabd56a47ec82c0ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732776938589 2024-11-28T06:55:45,896 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d10f70d03b8492882922f730b75029e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732776938589 2024-11-28T06:55:45,897 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b56fb8e89504bbba6e5f3be3d28aff3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732776940816 2024-11-28T06:55:45,897 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 905117eab9b444bb867425c2e97958ed, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732776940816 2024-11-28T06:55:45,897 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c22987c0f814dd9a651e61112f03805, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732776943042 2024-11-28T06:55:45,897 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25b07c176652456fa6f1e8dd3fce48cc, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732776943022 2024-11-28T06:55:45,906 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a23eb2f63cfcb5c9c85b72f46ebe560c] 2024-11-28T06:55:45,908 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411282e6a7b70287a47a9a4c557906f36e62c_a23eb2f63cfcb5c9c85b72f46ebe560c store=[table=TestAcidGuarantees family=A region=a23eb2f63cfcb5c9c85b72f46ebe560c] 2024-11-28T06:55:45,908 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a23eb2f63cfcb5c9c85b72f46ebe560c#B#compaction#295 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:55:45,908 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/5b6c9605bfda4e3c921d06800481bf93 is 50, key is test_row_0/B:col10/1732776944201/Put/seqid=0 2024-11-28T06:55:45,911 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411282e6a7b70287a47a9a4c557906f36e62c_a23eb2f63cfcb5c9c85b72f46ebe560c, store=[table=TestAcidGuarantees family=A region=a23eb2f63cfcb5c9c85b72f46ebe560c] 2024-11-28T06:55:45,911 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411282e6a7b70287a47a9a4c557906f36e62c_a23eb2f63cfcb5c9c85b72f46ebe560c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a23eb2f63cfcb5c9c85b72f46ebe560c] 2024-11-28T06:55:45,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742179_1355 (size=12697) 2024-11-28T06:55:45,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742180_1356 (size=4469) 2024-11-28T06:55:46,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-28T06:55:46,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:46,032 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing a23eb2f63cfcb5c9c85b72f46ebe560c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T06:55:46,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=A 2024-11-28T06:55:46,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:46,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=B 2024-11-28T06:55:46,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:46,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=C 2024-11-28T06:55:46,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:46,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411284eca679cf7334fff9151bec194778736_a23eb2f63cfcb5c9c85b72f46ebe560c is 50, key is test_row_0/A:col10/1732776944239/Put/seqid=0 2024-11-28T06:55:46,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742181_1357 (size=12304) 2024-11-28T06:55:46,317 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/5b6c9605bfda4e3c921d06800481bf93 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/5b6c9605bfda4e3c921d06800481bf93 2024-11-28T06:55:46,322 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a23eb2f63cfcb5c9c85b72f46ebe560c/B of a23eb2f63cfcb5c9c85b72f46ebe560c into 5b6c9605bfda4e3c921d06800481bf93(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:55:46,323 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a23eb2f63cfcb5c9c85b72f46ebe560c: 2024-11-28T06:55:46,323 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., storeName=a23eb2f63cfcb5c9c85b72f46ebe560c/B, priority=12, startTime=1732776945894; duration=0sec 2024-11-28T06:55:46,323 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:55:46,323 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a23eb2f63cfcb5c9c85b72f46ebe560c:B 2024-11-28T06:55:46,323 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T06:55:46,325 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T06:55:46,325 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): a23eb2f63cfcb5c9c85b72f46ebe560c/C is initiating minor compaction (all files) 2024-11-28T06:55:46,325 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a23eb2f63cfcb5c9c85b72f46ebe560c/C in TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:46,325 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a23eb2f63cfcb5c9c85b72f46ebe560c#A#compaction#296 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:55:46,325 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/26cdd831f5714ac2a9bc413d5fe48543, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/8268f0a32d154965b5249f5fc63a24ad, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/bc249dfb6ca24e84827e524e83604133, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/6baa231a50474600bc96fa4244f0c727] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp, totalSize=47.9 K 2024-11-28T06:55:46,325 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/846ac3da5a63400dac0ae636ded65bac is 175, key is test_row_0/A:col10/1732776944201/Put/seqid=0 2024-11-28T06:55:46,326 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 26cdd831f5714ac2a9bc413d5fe48543, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732776937343 2024-11-28T06:55:46,326 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 8268f0a32d154965b5249f5fc63a24ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732776938589 2024-11-28T06:55:46,327 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting bc249dfb6ca24e84827e524e83604133, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732776940816 2024-11-28T06:55:46,327 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 6baa231a50474600bc96fa4244f0c727, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732776943042 2024-11-28T06:55:46,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742182_1358 (size=31651) 2024-11-28T06:55:46,339 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a23eb2f63cfcb5c9c85b72f46ebe560c#C#compaction#298 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:55:46,339 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/c67f54b520634afb928d1c8b90f241bd is 50, key is test_row_0/C:col10/1732776944201/Put/seqid=0 2024-11-28T06:55:46,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742183_1359 (size=12697) 2024-11-28T06:55:46,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:55:46,446 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411284eca679cf7334fff9151bec194778736_a23eb2f63cfcb5c9c85b72f46ebe560c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411284eca679cf7334fff9151bec194778736_a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:46,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/6bda3345d0534693bdbebe36a3ca6e0f, store: [table=TestAcidGuarantees family=A region=a23eb2f63cfcb5c9c85b72f46ebe560c] 2024-11-28T06:55:46,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/6bda3345d0534693bdbebe36a3ca6e0f is 175, key is test_row_0/A:col10/1732776944239/Put/seqid=0 2024-11-28T06:55:46,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:46,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. as already flushing 2024-11-28T06:55:46,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742184_1360 (size=31105) 2024-11-28T06:55:46,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:46,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732777006514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:46,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:46,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:46,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:46,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:46,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732777006518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:46,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:46,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:46,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:46,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:46,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732777006625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:46,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:46,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:46,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:46,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:46,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732777006636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:46,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:46,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:46,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:46,733 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/846ac3da5a63400dac0ae636ded65bac as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/846ac3da5a63400dac0ae636ded65bac 2024-11-28T06:55:46,737 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a23eb2f63cfcb5c9c85b72f46ebe560c/A of a23eb2f63cfcb5c9c85b72f46ebe560c into 846ac3da5a63400dac0ae636ded65bac(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:55:46,737 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a23eb2f63cfcb5c9c85b72f46ebe560c: 2024-11-28T06:55:46,737 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., storeName=a23eb2f63cfcb5c9c85b72f46ebe560c/A, priority=12, startTime=1732776945894; duration=0sec 2024-11-28T06:55:46,737 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:46,737 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a23eb2f63cfcb5c9c85b72f46ebe560c:A 2024-11-28T06:55:46,749 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/c67f54b520634afb928d1c8b90f241bd as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/c67f54b520634afb928d1c8b90f241bd 2024-11-28T06:55:46,753 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a23eb2f63cfcb5c9c85b72f46ebe560c/C of a23eb2f63cfcb5c9c85b72f46ebe560c into c67f54b520634afb928d1c8b90f241bd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:55:46,753 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a23eb2f63cfcb5c9c85b72f46ebe560c: 2024-11-28T06:55:46,753 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., storeName=a23eb2f63cfcb5c9c85b72f46ebe560c/C, priority=12, startTime=1732776945895; duration=0sec 2024-11-28T06:55:46,754 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:46,754 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a23eb2f63cfcb5c9c85b72f46ebe560c:C 2024-11-28T06:55:46,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-28T06:55:46,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:46,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732777006835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:46,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:46,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:46,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:46,859 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=250, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/6bda3345d0534693bdbebe36a3ca6e0f 2024-11-28T06:55:46,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:46,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732777006856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:46,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:46,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:46,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:46,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/acfd9f5d03c3484db96647b30cf3ac29 is 50, key is test_row_0/B:col10/1732776944239/Put/seqid=0 2024-11-28T06:55:46,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742185_1361 (size=12151) 2024-11-28T06:55:47,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:47,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732777007155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:47,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:47,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:47,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:47,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:47,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732777007165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:47,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:47,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:47,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:47,271 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/acfd9f5d03c3484db96647b30cf3ac29 2024-11-28T06:55:47,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/c433cd3205e54bbf9c248dbb061b65b4 is 50, key is test_row_0/C:col10/1732776944239/Put/seqid=0 2024-11-28T06:55:47,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742186_1362 (size=12151) 2024-11-28T06:55:47,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40092 deadline: 1732777007666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:47,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:47,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:47,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:47,684 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/c433cd3205e54bbf9c248dbb061b65b4 2024-11-28T06:55:47,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/6bda3345d0534693bdbebe36a3ca6e0f as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/6bda3345d0534693bdbebe36a3ca6e0f 2024-11-28T06:55:47,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40080 deadline: 1732777007685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:47,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:47,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:47,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:55:47,693 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/6bda3345d0534693bdbebe36a3ca6e0f, entries=150, sequenceid=250, filesize=30.4 K 2024-11-28T06:55:47,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/acfd9f5d03c3484db96647b30cf3ac29 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/acfd9f5d03c3484db96647b30cf3ac29 2024-11-28T06:55:47,697 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/acfd9f5d03c3484db96647b30cf3ac29, entries=150, sequenceid=250, filesize=11.9 K 2024-11-28T06:55:47,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/c433cd3205e54bbf9c248dbb061b65b4 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/c433cd3205e54bbf9c248dbb061b65b4 2024-11-28T06:55:47,699 INFO [Thread-1367 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1367. 2024-11-28T06:55:47,699 DEBUG [Thread-1367 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:55:47,700 DEBUG [Thread-1367 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:47,700 DEBUG [Thread-1367 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:47,700 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:55:47,701 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/c433cd3205e54bbf9c248dbb061b65b4, entries=150, sequenceid=250, filesize=11.9 K 2024-11-28T06:55:47,701 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for a23eb2f63cfcb5c9c85b72f46ebe560c in 1669ms, sequenceid=250, compaction requested=false 2024-11-28T06:55:47,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for a23eb2f63cfcb5c9c85b72f46ebe560c: 2024-11-28T06:55:47,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:47,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-28T06:55:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-28T06:55:47,704 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-28T06:55:47,704 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0540 sec 2024-11-28T06:55:47,704 INFO [Thread-1369 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1369. 2024-11-28T06:55:47,704 DEBUG [Thread-1369 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:55:47,704 DEBUG [Thread-1369 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:47,705 DEBUG [Thread-1369 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:47,705 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:55:47,706 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 3.0580 sec 2024-11-28T06:55:47,707 INFO [Thread-1373 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1373. 2024-11-28T06:55:47,707 DEBUG [Thread-1373 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:55:47,707 DEBUG [Thread-1373 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:47,707 DEBUG [Thread-1373 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:47,707 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:55:47,708 INFO [Thread-1371 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1371. 2024-11-28T06:55:47,708 DEBUG [Thread-1371 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:55:47,708 DEBUG [Thread-1371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:47,708 DEBUG [Thread-1371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:47,708 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:55:47,712 INFO [Thread-1365 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1365. 2024-11-28T06:55:47,712 DEBUG [Thread-1365 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:55:47,712 DEBUG [Thread-1365 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:47,712 DEBUG [Thread-1365 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:47,712 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:55:48,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:48,141 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a23eb2f63cfcb5c9c85b72f46ebe560c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T06:55:48,141 INFO [Thread-1356 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1356. 2024-11-28T06:55:48,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=A 2024-11-28T06:55:48,141 DEBUG [Thread-1356 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:55:48,141 DEBUG [Thread-1356 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:48,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:48,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=B 2024-11-28T06:55:48,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:48,141 DEBUG [Thread-1356 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:48,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=C 2024-11-28T06:55:48,141 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:55:48,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:48,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411282f80822696cc44519afe44328f000e0c_a23eb2f63cfcb5c9c85b72f46ebe560c is 50, key is test_row_0/A:col10/1732776946506/Put/seqid=0 2024-11-28T06:55:48,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742187_1363 (size=12454) 2024-11-28T06:55:48,177 INFO [Thread-1362 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1362. 2024-11-28T06:55:48,177 DEBUG [Thread-1362 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:55:48,177 DEBUG [Thread-1362 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:48,177 DEBUG [Thread-1362 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:48,177 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:55:48,188 INFO [Thread-1354 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1354. 2024-11-28T06:55:48,188 DEBUG [Thread-1354 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:55:48,188 DEBUG [Thread-1354 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:48,188 DEBUG [Thread-1354 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:48,188 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:55:48,556 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:55:48,565 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411282f80822696cc44519afe44328f000e0c_a23eb2f63cfcb5c9c85b72f46ebe560c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411282f80822696cc44519afe44328f000e0c_a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:48,567 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/d530d06b988e4285bcf985125603a848, store: [table=TestAcidGuarantees family=A region=a23eb2f63cfcb5c9c85b72f46ebe560c] 2024-11-28T06:55:48,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/d530d06b988e4285bcf985125603a848 is 175, key is test_row_0/A:col10/1732776946506/Put/seqid=0 2024-11-28T06:55:48,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742188_1364 (size=31255) 2024-11-28T06:55:48,687 INFO [Thread-1358 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1358. 2024-11-28T06:55:48,687 DEBUG [Thread-1358 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:55:48,687 DEBUG [Thread-1358 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:48,687 DEBUG [Thread-1358 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:48,687 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:55:48,707 INFO [Thread-1360 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1360. 2024-11-28T06:55:48,707 DEBUG [Thread-1360 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:55:48,707 DEBUG [Thread-1360 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:48,708 DEBUG [Thread-1360 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:48,708 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:55:48,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-28T06:55:48,806 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:55:48,806 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-28T06:55:48,806 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-28T06:55:48,806 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-28T06:55:48,806 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-11-28T06:55:48,806 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-28T06:55:48,806 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-11-28T06:55:48,806 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-28T06:55:48,806 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-28T06:55:48,806 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1083 2024-11-28T06:55:48,807 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3248 rows 2024-11-28T06:55:48,807 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1079 2024-11-28T06:55:48,807 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3235 rows 2024-11-28T06:55:48,807 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1082 2024-11-28T06:55:48,807 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3244 rows 2024-11-28T06:55:48,807 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1082 2024-11-28T06:55:48,807 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3246 rows 2024-11-28T06:55:48,807 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1084 2024-11-28T06:55:48,807 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3252 rows 2024-11-28T06:55:48,807 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T06:55:48,807 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T06:55:48,807 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:48,808 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:48,808 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:55:48,809 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-28T06:55:48,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-28T06:55:48,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-28T06:55:48,815 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732776948815"}]},"ts":"1732776948815"} 2024-11-28T06:55:48,817 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-28T06:55:48,817 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-28T06:55:48,818 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T06:55:48,820 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a23eb2f63cfcb5c9c85b72f46ebe560c, UNASSIGN}] 2024-11-28T06:55:48,821 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a23eb2f63cfcb5c9c85b72f46ebe560c, UNASSIGN 2024-11-28T06:55:48,822 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=103 updating hbase:meta row=a23eb2f63cfcb5c9c85b72f46ebe560c, regionState=CLOSING, regionLocation=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:48,824 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a23eb2f63cfcb5c9c85b72f46ebe560c, UNASSIGN because future has completed 2024-11-28T06:55:48,824 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T06:55:48,824 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; CloseRegionProcedure a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568}] 2024-11-28T06:55:48,889 WARN [regionserver/1cb9cf437a36:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-28T06:55:48,890 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/edf4cd3943584a03bfd7325972857f59, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/7595faaf5e04498f8ea538614292f7d8, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/3e115d0567324cdab8cb58fa806836ae, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/631488c4571b4a18a7dd1c7db98ecea5, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/a1087292fa2b47e08a212d6aaacc0e98, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/a38e26179add4c799199dc8fb82b6979, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/ecbdca11b8004e32bcb6a37dec8cd187, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/3a19a250e1704ab4987960427be204be, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/c338a105e94c4c07bd6bc0ec8faaeabc, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/a9c1b4a5b7cb469e917c3ee92694e497, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/cc03d86d010e4ea2aa958c82e26cb252, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/e458197de130470cad31676ad877c68f, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/1ad12571d0c1433084422901f244bb0a, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/1d10f70d03b8492882922f730b75029e, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/905117eab9b444bb867425c2e97958ed, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/25b07c176652456fa6f1e8dd3fce48cc] to archive 2024-11-28T06:55:48,891 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T06:55:48,892 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/edf4cd3943584a03bfd7325972857f59 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/edf4cd3943584a03bfd7325972857f59 2024-11-28T06:55:48,894 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/7595faaf5e04498f8ea538614292f7d8 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/7595faaf5e04498f8ea538614292f7d8 2024-11-28T06:55:48,895 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/3e115d0567324cdab8cb58fa806836ae to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/3e115d0567324cdab8cb58fa806836ae 2024-11-28T06:55:48,896 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/631488c4571b4a18a7dd1c7db98ecea5 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/631488c4571b4a18a7dd1c7db98ecea5 2024-11-28T06:55:48,896 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/a1087292fa2b47e08a212d6aaacc0e98 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/a1087292fa2b47e08a212d6aaacc0e98 2024-11-28T06:55:48,897 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/a38e26179add4c799199dc8fb82b6979 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/a38e26179add4c799199dc8fb82b6979 2024-11-28T06:55:48,898 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/ecbdca11b8004e32bcb6a37dec8cd187 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/ecbdca11b8004e32bcb6a37dec8cd187 2024-11-28T06:55:48,899 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/3a19a250e1704ab4987960427be204be to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/3a19a250e1704ab4987960427be204be 2024-11-28T06:55:48,900 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/c338a105e94c4c07bd6bc0ec8faaeabc to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/c338a105e94c4c07bd6bc0ec8faaeabc 2024-11-28T06:55:48,901 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/a9c1b4a5b7cb469e917c3ee92694e497 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/a9c1b4a5b7cb469e917c3ee92694e497 2024-11-28T06:55:48,901 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/cc03d86d010e4ea2aa958c82e26cb252 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/cc03d86d010e4ea2aa958c82e26cb252 2024-11-28T06:55:48,902 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/e458197de130470cad31676ad877c68f to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/e458197de130470cad31676ad877c68f 2024-11-28T06:55:48,903 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/1ad12571d0c1433084422901f244bb0a to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/1ad12571d0c1433084422901f244bb0a 2024-11-28T06:55:48,904 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/1d10f70d03b8492882922f730b75029e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/1d10f70d03b8492882922f730b75029e 2024-11-28T06:55:48,905 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/905117eab9b444bb867425c2e97958ed to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/905117eab9b444bb867425c2e97958ed 2024-11-28T06:55:48,905 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/25b07c176652456fa6f1e8dd3fce48cc to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/25b07c176652456fa6f1e8dd3fce48cc 2024-11-28T06:55:48,907 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/6260b1ba85dd4d18babcb0235574f5a7, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/28e4614c6a874c309e512a7bc7ef84a8, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/b00c3b10e7f343e4bc8541d5d2671fb0, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/b3526ad723c5451dba69debebcd36600, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/f556c8fe93824216bfa2dd923d7fc35b, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/01ac587c37344178ba0247f8874c48cc, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/cfed94df144d4d67ae259e63d864dcda, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/bd2a76c4ae4f4615b14714428b615121, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/99993b8b93ca47478de4ee5bbdc622c5, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/832ddc81fec14e039b3279ca41eef084, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/178c80765bd7483ea85cdb04addd790c, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/76f8ff50c5294e608fbaa00161200402, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/01f837ec6b9c475ea21d817324f85848, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/a78f1d6bcc83401aabd56a47ec82c0ee, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/7b56fb8e89504bbba6e5f3be3d28aff3, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/4c22987c0f814dd9a651e61112f03805] to archive 2024-11-28T06:55:48,908 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T06:55:48,909 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/6260b1ba85dd4d18babcb0235574f5a7 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/6260b1ba85dd4d18babcb0235574f5a7 2024-11-28T06:55:48,910 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/28e4614c6a874c309e512a7bc7ef84a8 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/28e4614c6a874c309e512a7bc7ef84a8 2024-11-28T06:55:48,911 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/b00c3b10e7f343e4bc8541d5d2671fb0 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/b00c3b10e7f343e4bc8541d5d2671fb0 2024-11-28T06:55:48,912 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/b3526ad723c5451dba69debebcd36600 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/b3526ad723c5451dba69debebcd36600 2024-11-28T06:55:48,912 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/f556c8fe93824216bfa2dd923d7fc35b to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/f556c8fe93824216bfa2dd923d7fc35b 2024-11-28T06:55:48,913 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/01ac587c37344178ba0247f8874c48cc to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/01ac587c37344178ba0247f8874c48cc 2024-11-28T06:55:48,914 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/cfed94df144d4d67ae259e63d864dcda to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/cfed94df144d4d67ae259e63d864dcda 2024-11-28T06:55:48,915 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/bd2a76c4ae4f4615b14714428b615121 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/bd2a76c4ae4f4615b14714428b615121 2024-11-28T06:55:48,915 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/99993b8b93ca47478de4ee5bbdc622c5 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/99993b8b93ca47478de4ee5bbdc622c5 2024-11-28T06:55:48,916 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/832ddc81fec14e039b3279ca41eef084 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/832ddc81fec14e039b3279ca41eef084 2024-11-28T06:55:48,917 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/178c80765bd7483ea85cdb04addd790c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/178c80765bd7483ea85cdb04addd790c 2024-11-28T06:55:48,918 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/76f8ff50c5294e608fbaa00161200402 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/76f8ff50c5294e608fbaa00161200402 2024-11-28T06:55:48,919 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/01f837ec6b9c475ea21d817324f85848 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/01f837ec6b9c475ea21d817324f85848 2024-11-28T06:55:48,919 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/a78f1d6bcc83401aabd56a47ec82c0ee to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/a78f1d6bcc83401aabd56a47ec82c0ee 2024-11-28T06:55:48,920 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/7b56fb8e89504bbba6e5f3be3d28aff3 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/7b56fb8e89504bbba6e5f3be3d28aff3 2024-11-28T06:55:48,921 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/4c22987c0f814dd9a651e61112f03805 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/4c22987c0f814dd9a651e61112f03805 2024-11-28T06:55:48,923 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/3c6f4b1fd2d448ed90cd26cd867bc487, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/7a2fa727256143189af7fa469c31de7e, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/1be44fd891304e3dab7129938f193889, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/e75f3b52251d40e38219183f5a4e5590, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/44be8c6e67594af89b7f9216e130abd2, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/25cc27fd9c054b7d8f9540efea100992, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/cc55df6889634495b3df31ffea6392db, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/a37a7cfb28b648e78d805405786b3022, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/29523127218e41e28b5782ca8a6240b8, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/651d1981e0024c5fa8f4c2351be2b01a, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/9dbdd395d2c34eb1ba247783b2e7ff63, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/26cdd831f5714ac2a9bc413d5fe48543, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/386e78186621453d990711fc8fd75c58, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/8268f0a32d154965b5249f5fc63a24ad, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/bc249dfb6ca24e84827e524e83604133, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/6baa231a50474600bc96fa4244f0c727] to archive 2024-11-28T06:55:48,923 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T06:55:48,924 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/3c6f4b1fd2d448ed90cd26cd867bc487 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/3c6f4b1fd2d448ed90cd26cd867bc487 2024-11-28T06:55:48,925 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/7a2fa727256143189af7fa469c31de7e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/7a2fa727256143189af7fa469c31de7e 2024-11-28T06:55:48,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-28T06:55:48,926 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/1be44fd891304e3dab7129938f193889 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/1be44fd891304e3dab7129938f193889 2024-11-28T06:55:48,927 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/e75f3b52251d40e38219183f5a4e5590 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/e75f3b52251d40e38219183f5a4e5590 2024-11-28T06:55:48,927 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/44be8c6e67594af89b7f9216e130abd2 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/44be8c6e67594af89b7f9216e130abd2 2024-11-28T06:55:48,928 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/25cc27fd9c054b7d8f9540efea100992 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/25cc27fd9c054b7d8f9540efea100992 2024-11-28T06:55:48,929 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/cc55df6889634495b3df31ffea6392db to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/cc55df6889634495b3df31ffea6392db 2024-11-28T06:55:48,929 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/a37a7cfb28b648e78d805405786b3022 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/a37a7cfb28b648e78d805405786b3022 2024-11-28T06:55:48,930 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/29523127218e41e28b5782ca8a6240b8 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/29523127218e41e28b5782ca8a6240b8 2024-11-28T06:55:48,931 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/651d1981e0024c5fa8f4c2351be2b01a to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/651d1981e0024c5fa8f4c2351be2b01a 2024-11-28T06:55:48,932 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/9dbdd395d2c34eb1ba247783b2e7ff63 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/9dbdd395d2c34eb1ba247783b2e7ff63 2024-11-28T06:55:48,932 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/26cdd831f5714ac2a9bc413d5fe48543 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/26cdd831f5714ac2a9bc413d5fe48543 2024-11-28T06:55:48,933 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/386e78186621453d990711fc8fd75c58 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/386e78186621453d990711fc8fd75c58 2024-11-28T06:55:48,934 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/8268f0a32d154965b5249f5fc63a24ad to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/8268f0a32d154965b5249f5fc63a24ad 2024-11-28T06:55:48,935 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/bc249dfb6ca24e84827e524e83604133 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/bc249dfb6ca24e84827e524e83604133 2024-11-28T06:55:48,935 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/6baa231a50474600bc96fa4244f0c727 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/6baa231a50474600bc96fa4244f0c727 2024-11-28T06:55:48,972 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=272, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/d530d06b988e4285bcf985125603a848 2024-11-28T06:55:48,977 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(122): Close a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:48,977 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T06:55:48,977 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1722): Closing a23eb2f63cfcb5c9c85b72f46ebe560c, disabling compactions & flushes 2024-11-28T06:55:48,978 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:48,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/12d3b53ea0be4d429c8b32a8ac0d6571 is 50, key is test_row_0/B:col10/1732776946506/Put/seqid=0 2024-11-28T06:55:48,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742189_1365 (size=12301) 2024-11-28T06:55:49,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-28T06:55:49,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/12d3b53ea0be4d429c8b32a8ac0d6571 2024-11-28T06:55:49,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/72d3a2aea84c4c0c815ece8a155e9823 is 50, key is test_row_0/C:col10/1732776946506/Put/seqid=0 2024-11-28T06:55:49,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742190_1366 (size=12301) 2024-11-28T06:55:49,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-28T06:55:49,804 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/72d3a2aea84c4c0c815ece8a155e9823 2024-11-28T06:55:49,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/d530d06b988e4285bcf985125603a848 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/d530d06b988e4285bcf985125603a848 2024-11-28T06:55:49,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/d530d06b988e4285bcf985125603a848, entries=150, sequenceid=272, filesize=30.5 K 2024-11-28T06:55:49,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/12d3b53ea0be4d429c8b32a8ac0d6571 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/12d3b53ea0be4d429c8b32a8ac0d6571 2024-11-28T06:55:49,819 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/12d3b53ea0be4d429c8b32a8ac0d6571, entries=150, sequenceid=272, filesize=12.0 K 2024-11-28T06:55:49,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/72d3a2aea84c4c0c815ece8a155e9823 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/72d3a2aea84c4c0c815ece8a155e9823 2024-11-28T06:55:49,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/72d3a2aea84c4c0c815ece8a155e9823, entries=150, sequenceid=272, filesize=12.0 K 2024-11-28T06:55:49,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=26.84 KB/27480 for a23eb2f63cfcb5c9c85b72f46ebe560c in 1684ms, sequenceid=272, compaction requested=true 2024-11-28T06:55:49,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a23eb2f63cfcb5c9c85b72f46ebe560c: 2024-11-28T06:55:49,824 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:49,824 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:49,824 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. because compaction request was cancelled 2024-11-28T06:55:49,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a23eb2f63cfcb5c9c85b72f46ebe560c:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:55:49,824 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a23eb2f63cfcb5c9c85b72f46ebe560c:A 2024-11-28T06:55:49,824 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. after waiting 0 ms 2024-11-28T06:55:49,824 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:49,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:49,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a23eb2f63cfcb5c9c85b72f46ebe560c:B, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:55:49,824 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. because compaction request was cancelled 2024-11-28T06:55:49,824 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(2902): Flushing a23eb2f63cfcb5c9c85b72f46ebe560c 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-28T06:55:49,824 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a23eb2f63cfcb5c9c85b72f46ebe560c:B 2024-11-28T06:55:49,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:49,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a23eb2f63cfcb5c9c85b72f46ebe560c:C, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:55:49,825 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. because compaction request was cancelled 2024-11-28T06:55:49,825 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a23eb2f63cfcb5c9c85b72f46ebe560c:C 2024-11-28T06:55:49,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:55:49,825 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=A 2024-11-28T06:55:49,825 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:49,825 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=B 2024-11-28T06:55:49,825 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:49,825 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a23eb2f63cfcb5c9c85b72f46ebe560c, store=C 2024-11-28T06:55:49,825 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:49,829 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128f145715ca0244d17b4d0a640ab7bc607_a23eb2f63cfcb5c9c85b72f46ebe560c is 50, key is test_row_0/A:col10/1732776948706/Put/seqid=0 2024-11-28T06:55:49,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742191_1367 (size=12454) 2024-11-28T06:55:49,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-28T06:55:50,233 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:55:50,240 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128f145715ca0244d17b4d0a640ab7bc607_a23eb2f63cfcb5c9c85b72f46ebe560c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128f145715ca0244d17b4d0a640ab7bc607_a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:50,242 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/1e8ada98a838493eaf3adf9e925efbb8, store: [table=TestAcidGuarantees family=A region=a23eb2f63cfcb5c9c85b72f46ebe560c] 2024-11-28T06:55:50,242 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/1e8ada98a838493eaf3adf9e925efbb8 is 175, key is test_row_0/A:col10/1732776948706/Put/seqid=0 2024-11-28T06:55:50,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742192_1368 (size=31255) 2024-11-28T06:55:50,649 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=279, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/1e8ada98a838493eaf3adf9e925efbb8 2024-11-28T06:55:50,655 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/9f51125eee4d4c518ee21d24e861e3c1 is 50, key is test_row_0/B:col10/1732776948706/Put/seqid=0 2024-11-28T06:55:50,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742193_1369 (size=12301) 2024-11-28T06:55:50,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-28T06:55:51,059 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/9f51125eee4d4c518ee21d24e861e3c1 2024-11-28T06:55:51,065 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/9ad6c84fdf7b4605bb505ea4370c62d6 is 50, key is test_row_0/C:col10/1732776948706/Put/seqid=0 2024-11-28T06:55:51,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742194_1370 (size=12301) 2024-11-28T06:55:51,470 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/9ad6c84fdf7b4605bb505ea4370c62d6 2024-11-28T06:55:51,479 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/A/1e8ada98a838493eaf3adf9e925efbb8 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/1e8ada98a838493eaf3adf9e925efbb8 2024-11-28T06:55:51,485 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/1e8ada98a838493eaf3adf9e925efbb8, entries=150, sequenceid=279, filesize=30.5 K 2024-11-28T06:55:51,486 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/B/9f51125eee4d4c518ee21d24e861e3c1 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/9f51125eee4d4c518ee21d24e861e3c1 2024-11-28T06:55:51,491 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/9f51125eee4d4c518ee21d24e861e3c1, entries=150, sequenceid=279, filesize=12.0 K 2024-11-28T06:55:51,492 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/.tmp/C/9ad6c84fdf7b4605bb505ea4370c62d6 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/9ad6c84fdf7b4605bb505ea4370c62d6 2024-11-28T06:55:51,497 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/9ad6c84fdf7b4605bb505ea4370c62d6, entries=150, sequenceid=279, filesize=12.0 K 2024-11-28T06:55:51,498 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for a23eb2f63cfcb5c9c85b72f46ebe560c in 1674ms, sequenceid=279, compaction requested=true 2024-11-28T06:55:51,503 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/recovered.edits/282.seqid, newMaxSeqId=282, maxSeqId=4 2024-11-28T06:55:51,503 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. 2024-11-28T06:55:51,503 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1676): Region close journal for a23eb2f63cfcb5c9c85b72f46ebe560c: Waiting for close lock at 1732776948977Running coprocessor pre-close hooks at 1732776948977Disabling compacts and flushes for region at 1732776948977Disabling writes for close at 1732776949824 (+847 ms)Obtaining lock to block concurrent updates at 1732776949824Preparing flush snapshotting stores in a23eb2f63cfcb5c9c85b72f46ebe560c at 1732776949824Finished memstore snapshotting TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732776949825 (+1 ms)Flushing stores of TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c. at 1732776949825Flushing a23eb2f63cfcb5c9c85b72f46ebe560c/A: creating writer at 1732776949825Flushing mob file a23eb2f63cfcb5c9c85b72f46ebe560c/A: appending metadata at 1732776949829 (+4 ms)Flushing mob file a23eb2f63cfcb5c9c85b72f46ebe560c/A: closing flushed file at 1732776949829Flushing a23eb2f63cfcb5c9c85b72f46ebe560c/A: appending metadata at 1732776950242 (+413 ms)Flushing a23eb2f63cfcb5c9c85b72f46ebe560c/A: closing flushed file at 1732776950242Flushing a23eb2f63cfcb5c9c85b72f46ebe560c/B: creating writer at 1732776950652 (+410 ms)Flushing a23eb2f63cfcb5c9c85b72f46ebe560c/B: appending metadata at 1732776950654 (+2 ms)Flushing a23eb2f63cfcb5c9c85b72f46ebe560c/B: closing flushed file at 1732776950654Flushing a23eb2f63cfcb5c9c85b72f46ebe560c/C: creating writer at 1732776951062 (+408 ms)Flushing a23eb2f63cfcb5c9c85b72f46ebe560c/C: appending metadata at 1732776951065 (+3 ms)Flushing a23eb2f63cfcb5c9c85b72f46ebe560c/C: closing flushed file at 1732776951065Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71da1bdd: reopening flushed file at 1732776951478 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55c4b7df: reopening flushed file at 1732776951485 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2edaa40b: reopening flushed file at 1732776951491 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for a23eb2f63cfcb5c9c85b72f46ebe560c in 1674ms, sequenceid=279, compaction requested=true at 1732776951498 (+7 ms)Writing region close event to WAL at 1732776951500 (+2 ms)Running coprocessor post-close hooks at 1732776951503 (+3 ms)Closed at 1732776951503 2024-11-28T06:55:51,505 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(157): Closed a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:51,506 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=103 updating hbase:meta row=a23eb2f63cfcb5c9c85b72f46ebe560c, regionState=CLOSED 2024-11-28T06:55:51,507 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=104, ppid=103, state=RUNNABLE, hasLock=false; CloseRegionProcedure a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 because future has completed 2024-11-28T06:55:51,509 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-28T06:55:51,509 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; CloseRegionProcedure a23eb2f63cfcb5c9c85b72f46ebe560c, server=1cb9cf437a36,43341,1732776825568 in 2.6830 sec 2024-11-28T06:55:51,510 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=103, resume processing ppid=102 2024-11-28T06:55:51,510 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, ppid=102, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a23eb2f63cfcb5c9c85b72f46ebe560c, UNASSIGN in 2.6890 sec 2024-11-28T06:55:51,512 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-28T06:55:51,512 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.6930 sec 2024-11-28T06:55:51,513 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732776951513"}]},"ts":"1732776951513"} 2024-11-28T06:55:51,514 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-28T06:55:51,514 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-28T06:55:51,516 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.7060 sec 2024-11-28T06:55:52,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-28T06:55:52,978 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:55:52,979 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-28T06:55:52,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T06:55:52,983 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=105, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T06:55:52,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-28T06:55:52,985 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=105, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T06:55:52,989 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:52,991 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A, FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B, FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C, FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/recovered.edits] 2024-11-28T06:55:52,994 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/1e8ada98a838493eaf3adf9e925efbb8 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/1e8ada98a838493eaf3adf9e925efbb8 2024-11-28T06:55:52,995 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/6bda3345d0534693bdbebe36a3ca6e0f to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/6bda3345d0534693bdbebe36a3ca6e0f 2024-11-28T06:55:52,996 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/846ac3da5a63400dac0ae636ded65bac to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/846ac3da5a63400dac0ae636ded65bac 2024-11-28T06:55:52,997 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/d530d06b988e4285bcf985125603a848 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/A/d530d06b988e4285bcf985125603a848 2024-11-28T06:55:52,998 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/12d3b53ea0be4d429c8b32a8ac0d6571 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/12d3b53ea0be4d429c8b32a8ac0d6571 2024-11-28T06:55:52,999 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/5b6c9605bfda4e3c921d06800481bf93 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/5b6c9605bfda4e3c921d06800481bf93 2024-11-28T06:55:53,000 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/9f51125eee4d4c518ee21d24e861e3c1 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/9f51125eee4d4c518ee21d24e861e3c1 2024-11-28T06:55:53,000 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/acfd9f5d03c3484db96647b30cf3ac29 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/B/acfd9f5d03c3484db96647b30cf3ac29 2024-11-28T06:55:53,002 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/72d3a2aea84c4c0c815ece8a155e9823 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/72d3a2aea84c4c0c815ece8a155e9823 2024-11-28T06:55:53,003 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/9ad6c84fdf7b4605bb505ea4370c62d6 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/9ad6c84fdf7b4605bb505ea4370c62d6 2024-11-28T06:55:53,003 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/c433cd3205e54bbf9c248dbb061b65b4 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/c433cd3205e54bbf9c248dbb061b65b4 2024-11-28T06:55:53,004 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/c67f54b520634afb928d1c8b90f241bd to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/C/c67f54b520634afb928d1c8b90f241bd 2024-11-28T06:55:53,006 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/recovered.edits/282.seqid to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c/recovered.edits/282.seqid 2024-11-28T06:55:53,006 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:53,006 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-28T06:55:53,006 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-28T06:55:53,007 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-28T06:55:53,009 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112808585c9d3c9b4496a1a02b483076b049_a23eb2f63cfcb5c9c85b72f46ebe560c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112808585c9d3c9b4496a1a02b483076b049_a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:53,010 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411280f66a4621b35450fb8d8e8a4ae6deb3c_a23eb2f63cfcb5c9c85b72f46ebe560c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411280f66a4621b35450fb8d8e8a4ae6deb3c_a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:53,011 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112813e0ff9d038b486499daea6848ab2b22_a23eb2f63cfcb5c9c85b72f46ebe560c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112813e0ff9d038b486499daea6848ab2b22_a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:53,011 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411282f80822696cc44519afe44328f000e0c_a23eb2f63cfcb5c9c85b72f46ebe560c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411282f80822696cc44519afe44328f000e0c_a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:53,012 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411284eca679cf7334fff9151bec194778736_a23eb2f63cfcb5c9c85b72f46ebe560c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411284eca679cf7334fff9151bec194778736_a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:53,013 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411285d131360e2a249488fa9471f4f81974a_a23eb2f63cfcb5c9c85b72f46ebe560c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411285d131360e2a249488fa9471f4f81974a_a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:53,013 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128763375aa6ab94be0bc0bb1177ab57367_a23eb2f63cfcb5c9c85b72f46ebe560c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128763375aa6ab94be0bc0bb1177ab57367_a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:53,014 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288851490c0d03434cae97ab4c7710aec7_a23eb2f63cfcb5c9c85b72f46ebe560c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288851490c0d03434cae97ab4c7710aec7_a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:53,015 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d9ac9db4aee64e2b9f9635bcc9e975e5_a23eb2f63cfcb5c9c85b72f46ebe560c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d9ac9db4aee64e2b9f9635bcc9e975e5_a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:53,016 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128dd91e6966a3d4334ad8b13da961f2da6_a23eb2f63cfcb5c9c85b72f46ebe560c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128dd91e6966a3d4334ad8b13da961f2da6_a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:53,017 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e0002ce346b7407797403e9d5bf47393_a23eb2f63cfcb5c9c85b72f46ebe560c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e0002ce346b7407797403e9d5bf47393_a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:53,018 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e81daac444bd44759403494da490bc8c_a23eb2f63cfcb5c9c85b72f46ebe560c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e81daac444bd44759403494da490bc8c_a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:53,018 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128f145715ca0244d17b4d0a640ab7bc607_a23eb2f63cfcb5c9c85b72f46ebe560c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128f145715ca0244d17b4d0a640ab7bc607_a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:53,019 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128f55cc711c61c4f22b5d224a3c1e1b48b_a23eb2f63cfcb5c9c85b72f46ebe560c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128f55cc711c61c4f22b5d224a3c1e1b48b_a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:53,020 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128fac978cbfeeb4ddd948a705d752f0ffa_a23eb2f63cfcb5c9c85b72f46ebe560c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128fac978cbfeeb4ddd948a705d752f0ffa_a23eb2f63cfcb5c9c85b72f46ebe560c 2024-11-28T06:55:53,020 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-28T06:55:53,022 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=105, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T06:55:53,024 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-28T06:55:53,026 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-28T06:55:53,028 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=105, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T06:55:53,028 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-28T06:55:53,028 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732776953028"}]},"ts":"9223372036854775807"} 2024-11-28T06:55:53,030 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-28T06:55:53,030 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a23eb2f63cfcb5c9c85b72f46ebe560c, NAME => 'TestAcidGuarantees,,1732776924554.a23eb2f63cfcb5c9c85b72f46ebe560c.', STARTKEY => '', ENDKEY => ''}] 2024-11-28T06:55:53,030 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-28T06:55:53,030 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732776953030"}]},"ts":"9223372036854775807"} 2024-11-28T06:55:53,031 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-28T06:55:53,032 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=105, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T06:55:53,033 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 53 msec 2024-11-28T06:55:53,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-28T06:55:53,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-28T06:55:53,099 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:55:53,108 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250 (was 248) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=416 (was 466), ProcessCount=11 (was 11), AvailableMemoryMB=7810 (was 8125) 2024-11-28T06:55:53,118 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=416, ProcessCount=11, AvailableMemoryMB=7810 2024-11-28T06:55:53,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T06:55:53,120 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T06:55:53,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=106, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-28T06:55:53,122 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T06:55:53,122 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:55:53,122 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 106 2024-11-28T06:55:53,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-28T06:55:53,123 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T06:55:53,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742195_1371 (size=1061) 2024-11-28T06:55:53,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-28T06:55:53,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-28T06:55:53,535 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7 2024-11-28T06:55:53,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742196_1372 (size=53) 2024-11-28T06:55:53,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-28T06:55:53,945 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T06:55:53,945 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 00878c8327d194b967221c1933e24c9a, disabling compactions & flushes 2024-11-28T06:55:53,945 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:53,945 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:53,945 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. after waiting 0 ms 2024-11-28T06:55:53,945 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:53,945 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:53,945 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 00878c8327d194b967221c1933e24c9a: Waiting for close lock at 1732776953945Disabling compacts and flushes for region at 1732776953945Disabling writes for close at 1732776953945Writing region close event to WAL at 1732776953945Closed at 1732776953945 2024-11-28T06:55:53,948 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T06:55:53,949 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732776953948"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732776953948"}]},"ts":"1732776953948"} 2024-11-28T06:55:53,951 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T06:55:53,952 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T06:55:53,952 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732776953952"}]},"ts":"1732776953952"} 2024-11-28T06:55:53,953 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-28T06:55:53,954 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00878c8327d194b967221c1933e24c9a, ASSIGN}] 2024-11-28T06:55:53,955 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00878c8327d194b967221c1933e24c9a, ASSIGN 2024-11-28T06:55:53,956 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=00878c8327d194b967221c1933e24c9a, ASSIGN; state=OFFLINE, location=1cb9cf437a36,43341,1732776825568; forceNewPlan=false, retain=false 2024-11-28T06:55:54,106 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=00878c8327d194b967221c1933e24c9a, regionState=OPENING, regionLocation=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:54,107 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=00878c8327d194b967221c1933e24c9a, ASSIGN because future has completed 2024-11-28T06:55:54,108 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568}] 2024-11-28T06:55:54,263 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:54,264 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7752): Opening region: {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} 2024-11-28T06:55:54,265 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 00878c8327d194b967221c1933e24c9a 2024-11-28T06:55:54,265 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T06:55:54,266 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7794): checking encryption for 00878c8327d194b967221c1933e24c9a 2024-11-28T06:55:54,266 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7797): checking classloading for 00878c8327d194b967221c1933e24c9a 2024-11-28T06:55:54,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-28T06:55:54,268 INFO [StoreOpener-00878c8327d194b967221c1933e24c9a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 00878c8327d194b967221c1933e24c9a 2024-11-28T06:55:54,269 INFO [StoreOpener-00878c8327d194b967221c1933e24c9a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T06:55:54,269 INFO [StoreOpener-00878c8327d194b967221c1933e24c9a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00878c8327d194b967221c1933e24c9a columnFamilyName A 2024-11-28T06:55:54,269 DEBUG [StoreOpener-00878c8327d194b967221c1933e24c9a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:55:54,270 INFO [StoreOpener-00878c8327d194b967221c1933e24c9a-1 {}] regionserver.HStore(327): Store=00878c8327d194b967221c1933e24c9a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:55:54,270 INFO [StoreOpener-00878c8327d194b967221c1933e24c9a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 00878c8327d194b967221c1933e24c9a 2024-11-28T06:55:54,270 INFO [StoreOpener-00878c8327d194b967221c1933e24c9a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T06:55:54,271 INFO [StoreOpener-00878c8327d194b967221c1933e24c9a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00878c8327d194b967221c1933e24c9a columnFamilyName B 2024-11-28T06:55:54,271 DEBUG [StoreOpener-00878c8327d194b967221c1933e24c9a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:55:54,271 INFO [StoreOpener-00878c8327d194b967221c1933e24c9a-1 {}] regionserver.HStore(327): Store=00878c8327d194b967221c1933e24c9a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:55:54,271 INFO [StoreOpener-00878c8327d194b967221c1933e24c9a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 00878c8327d194b967221c1933e24c9a 2024-11-28T06:55:54,272 INFO [StoreOpener-00878c8327d194b967221c1933e24c9a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T06:55:54,272 INFO [StoreOpener-00878c8327d194b967221c1933e24c9a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00878c8327d194b967221c1933e24c9a columnFamilyName C 2024-11-28T06:55:54,272 DEBUG [StoreOpener-00878c8327d194b967221c1933e24c9a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:55:54,272 INFO [StoreOpener-00878c8327d194b967221c1933e24c9a-1 {}] regionserver.HStore(327): Store=00878c8327d194b967221c1933e24c9a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:55:54,272 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:54,273 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1038): replaying wal for 00878c8327d194b967221c1933e24c9a 2024-11-28T06:55:54,273 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a 2024-11-28T06:55:54,274 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a 2024-11-28T06:55:54,274 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1048): stopping wal replay for 00878c8327d194b967221c1933e24c9a 2024-11-28T06:55:54,274 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1060): Cleaning up temporary data for 00878c8327d194b967221c1933e24c9a 2024-11-28T06:55:54,275 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T06:55:54,276 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1093): writing seq id for 00878c8327d194b967221c1933e24c9a 2024-11-28T06:55:54,278 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T06:55:54,279 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1114): Opened 00878c8327d194b967221c1933e24c9a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66605074, jitterRate=-0.007507055997848511}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T06:55:54,279 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 00878c8327d194b967221c1933e24c9a 2024-11-28T06:55:54,279 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1006): Region open journal for 00878c8327d194b967221c1933e24c9a: Running coprocessor pre-open hook at 1732776954266Writing region info on filesystem at 1732776954266Initializing all the Stores at 1732776954267 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732776954267Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732776954268 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732776954268Cleaning up temporary data from old regions at 1732776954275 (+7 ms)Running coprocessor post-open hooks at 1732776954279 (+4 ms)Region opened successfully at 1732776954279 2024-11-28T06:55:54,280 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., pid=108, masterSystemTime=1732776954259 2024-11-28T06:55:54,282 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:54,282 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:54,283 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=00878c8327d194b967221c1933e24c9a, regionState=OPEN, openSeqNum=2, regionLocation=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:54,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=108, ppid=107, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 because future has completed 2024-11-28T06:55:54,286 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-28T06:55:54,286 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; OpenRegionProcedure 00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 in 177 msec 2024-11-28T06:55:54,287 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=107, resume processing ppid=106 2024-11-28T06:55:54,287 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, ppid=106, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00878c8327d194b967221c1933e24c9a, ASSIGN in 333 msec 2024-11-28T06:55:54,288 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T06:55:54,288 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732776954288"}]},"ts":"1732776954288"} 2024-11-28T06:55:54,289 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-28T06:55:54,290 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T06:55:54,291 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1700 sec 2024-11-28T06:55:55,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-28T06:55:55,286 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:55:55,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72240ede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:55,289 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:55:55,289 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:55:55,291 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:55:55,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:55:55,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:55:55,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57ee39ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:55,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:55:55,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:55:55,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:55,293 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52688, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:55:55,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1409d9d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:55,294 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:55:55,295 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:55:55,295 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:55:55,296 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46352, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:55:55,297 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T06:55:55,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 1cb9cf437a36,37503,1732776824789 2024-11-28T06:55:55,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@78fd32b9 2024-11-28T06:55:55,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T06:55:55,299 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52702, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T06:55:55,301 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@503e1f3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:55,301 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:55:55,301 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:55:55,302 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:55:55,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:55:55,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:55:55,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72a65f77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:55,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:55:55,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:55:55,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:55,304 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52720, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:55:55,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@faf6972, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:55,306 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205b33d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:55,306 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:55:55,306 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:55:55,307 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:55:55,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:55:55,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:55:55,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ded6683, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:55,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:55:55,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:55:55,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:55,308 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52740, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:55:55,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e0b10d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:55,310 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e100fd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:55,310 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:55:55,310 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:55:55,311 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:55:55,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:55:55,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:55:55,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ca6d17b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:55,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:55:55,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:55:55,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:55,312 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52758, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:55:55,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7128db41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:55,313 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58e40afb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:55,313 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:55:55,313 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:55:55,314 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:55:55,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:55:55,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:55:55,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70b9af1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:55,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:55:55,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:55:55,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:55,316 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52772, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:55:55,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@570c9f26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:55,317 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b9f4da0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:55,317 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:55:55,317 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:55:55,318 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:55:55,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:55:55,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:55:55,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a524871, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:55,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:55:55,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:55:55,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:55,319 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52798, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:55:55,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c320843, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:55,319 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4cbf6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:55,319 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:55:55,320 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:55:55,320 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:55:55,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:55:55,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:55:55,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e4f9ee8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:55,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:55:55,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:55:55,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:55,321 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52806, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:55:55,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@436599fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:55,322 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c83334c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:55,322 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:55:55,322 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:55:55,323 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:55:55,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:55:55,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:55:55,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43bc8da2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:55,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:55:55,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:55:55,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:55,324 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52822, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:55:55,324 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b465c7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:55,325 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3532b5c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:55,325 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:55:55,325 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:55:55,325 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:55:55,325 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:55:55,325 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:55:55,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d2403df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:55,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:55:55,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:55:55,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:55,326 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52842, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:55:55,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a804bbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:55,327 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3088fab0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:55,327 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:55:55,327 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:55:55,328 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:55:55,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:55:55,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:55:55,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d88cc92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:55,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:55:55,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:55:55,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:55,329 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52858, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:55:55,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e716a76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:55,330 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@663bb5d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:55,330 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:55:55,330 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:55:55,331 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:55:55,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:55:55,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:55:55,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@eda7861, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:55,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:55:55,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:55:55,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:55:55,332 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52888, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:55:55,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33a3356f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:55:55,333 DEBUG [Thread-1724 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:55:55,333 DEBUG [Thread-1709 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:55:55,333 DEBUG [Thread-1720 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:55:55,333 DEBUG [Thread-1713 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:55:55,333 DEBUG [Thread-1711 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:55:55,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:55:55,334 DEBUG [Thread-1715 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:55:55,334 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:55:55,334 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:55:55,334 DEBUG [Thread-1722 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:55:55,334 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:55:55,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:55:55,334 DEBUG [Thread-1726 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:55:55,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:55:55,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:55:55,335 DEBUG [Thread-1715 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:55:55,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:55:55,335 DEBUG [Thread-1713 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:55:55,335 DEBUG [Thread-1715 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:55:55,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:55:55,335 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46374, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:55:55,335 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46364, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:55:55,336 DEBUG [Thread-1728 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:55:55,336 DEBUG [Thread-1722 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:55:55,336 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46388, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:55:55,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-28T06:55:55,336 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46398, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:55:55,336 DEBUG [Thread-1722 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:55:55,336 DEBUG [Thread-1717 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:55:55,336 DEBUG [Thread-1711 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:55:55,336 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46404, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:55:55,336 DEBUG [Thread-1711 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:55:55,337 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46406, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:55:55,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:55:55,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:55:55,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2] 2024-11-28T06:55:55,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-28T06:55:55,338 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46420, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:55:55,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:55:55,338 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46426, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:55:55,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:55:55,339 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:55:55,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2] 2024-11-28T06:55:55,335 DEBUG [Thread-1713 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:55:55,340 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:55:55,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:55:55,341 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46450, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:55:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2] 2024-11-28T06:55:55,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2] 2024-11-28T06:55:55,342 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2] 2024-11-28T06:55:55,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2] 2024-11-28T06:55:55,343 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46434, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:55:55,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2] 2024-11-28T06:55:55,343 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2] 2024-11-28T06:55:55,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2] 2024-11-28T06:55:55,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2] 2024-11-28T06:55:55,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 00878c8327d194b967221c1933e24c9a 2024-11-28T06:55:55,352 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00878c8327d194b967221c1933e24c9a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T06:55:55,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=A 2024-11-28T06:55:55,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:55,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=B 2024-11-28T06:55:55,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:55,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=C 2024-11-28T06:55:55,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:55,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/446d92d35d744e32b42232548450e6ee is 50, key is test_row_0/A:col10/1732776955348/Put/seqid=0 2024-11-28T06:55:55,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742197_1373 (size=12001) 2024-11-28T06:55:55,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:55,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46398 deadline: 1732777015409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:55,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:55,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46420 deadline: 1732777015412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:55,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732776955393, max timestamp=1732776955420Num uniques 859; 2024-11-28T06:55:55,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732776955393, max timestamp=1732776955420Num uniques 859; 2024-11-28T06:55:55,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T06:55:55,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T06:55:55,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732776955393, max timestamp=1732776955421Num uniques 859; 2024-11-28T06:55:55,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T06:55:55,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:55,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:55,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:55,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:55,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:55,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:55,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-28T06:55:55,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:55,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777015465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:55,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:55,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732777015466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:55,476 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:55,476 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:55,476 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:55,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:55,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:55,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:55,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:55,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777015467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:55,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:55,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:55,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:55,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T06:55:55,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:55,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:55:55,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:55,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:55,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:55,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:55,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46398 deadline: 1732777015536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:55,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:55,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:55,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:55,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46420 deadline: 1732777015536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:55,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:55,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:55,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:55,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777015585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:55,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:55,594 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732777015586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:55,594 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:55,594 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:55,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:55,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:55,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777015586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:55,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:55,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:55,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:55,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:55,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T06:55:55,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:55,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:55:55,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:55,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:55,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:55,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-28T06:55:55,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46398 deadline: 1732777015755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:55,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:55,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:55,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46420 deadline: 1732777015755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:55,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:55,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:55,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:55,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:55,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/446d92d35d744e32b42232548450e6ee 2024-11-28T06:55:55,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/68c1bdb8b69b4971ba5fd8f93e44e5d1 is 50, key is test_row_0/B:col10/1732776955348/Put/seqid=0 2024-11-28T06:55:55,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T06:55:55,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:55,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:55:55,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:55,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:55,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:55,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742198_1374 (size=12001) 2024-11-28T06:55:55,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:55,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777015806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:55,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:55,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732777015806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:55,813 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:55,813 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:55,813 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:55,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:55,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:55,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:55,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:55,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777015806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:55,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:55,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:55,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:55,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T06:55:55,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:55,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:55:55,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:55,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:55,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-28T06:55:56,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:56,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46398 deadline: 1732777016065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:56,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:56,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:56,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46420 deadline: 1732777016065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:56,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:56,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:56,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:56,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:56,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:56,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T06:55:56,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:56,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:55:56,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:56,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:56,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:56,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:56,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:56,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732777016116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:56,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:56,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:56,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:56,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:56,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777016125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:56,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:56,133 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:56,133 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:56,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777016125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:56,133 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:56,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:56,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:56,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:56,205 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/68c1bdb8b69b4971ba5fd8f93e44e5d1 2024-11-28T06:55:56,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/4f54c3c115164b24869330622a0d3412 is 50, key is test_row_0/C:col10/1732776955348/Put/seqid=0 2024-11-28T06:55:56,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742199_1375 (size=12001) 2024-11-28T06:55:56,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T06:55:56,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:56,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:55:56,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:56,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:56,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:56,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:56,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T06:55:56,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:56,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:55:56,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:56,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:56,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:56,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:56,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-28T06:55:56,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T06:55:56,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:56,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:55:56,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:56,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:56,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:56,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:56,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:56,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46398 deadline: 1732777016576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:56,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:56,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:56,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:56,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:56,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46420 deadline: 1732777016585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:56,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:56,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:56,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:56,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/4f54c3c115164b24869330622a0d3412 2024-11-28T06:55:56,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/446d92d35d744e32b42232548450e6ee as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/446d92d35d744e32b42232548450e6ee 2024-11-28T06:55:56,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:56,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732777016625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:56,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:56,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:56,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:56,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/446d92d35d744e32b42232548450e6ee, entries=150, sequenceid=12, filesize=11.7 K 2024-11-28T06:55:56,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/68c1bdb8b69b4971ba5fd8f93e44e5d1 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/68c1bdb8b69b4971ba5fd8f93e44e5d1 2024-11-28T06:55:56,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/68c1bdb8b69b4971ba5fd8f93e44e5d1, entries=150, sequenceid=12, filesize=11.7 K 2024-11-28T06:55:56,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/4f54c3c115164b24869330622a0d3412 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/4f54c3c115164b24869330622a0d3412 2024-11-28T06:55:56,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/4f54c3c115164b24869330622a0d3412, entries=150, sequenceid=12, filesize=11.7 K 2024-11-28T06:55:56,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 00878c8327d194b967221c1933e24c9a in 1289ms, sequenceid=12, compaction requested=false 2024-11-28T06:55:56,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00878c8327d194b967221c1933e24c9a: 2024-11-28T06:55:56,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 00878c8327d194b967221c1933e24c9a 2024-11-28T06:55:56,652 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00878c8327d194b967221c1933e24c9a 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-28T06:55:56,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=A 2024-11-28T06:55:56,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:55:56,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=B 2024-11-28T06:55:56,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:55:56,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=C 2024-11-28T06:55:56,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:55:56,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/76189c5406404a54be4ec05e15e65f12 is 50, key is test_row_0/A:col10/1732776956650/Put/seqid=0 2024-11-28T06:55:56,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742200_1376 (size=14341) 2024-11-28T06:55:56,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:56,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777016683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:56,690 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:56,690 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:56,690 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:56,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:56,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777016684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:56,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:56,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:56,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:56,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T06:55:56,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:56,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:55:56,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:56,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:56,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:56,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:56,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:56,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777016795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:56,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:56,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777016795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:56,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:56,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:56,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:56,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T06:55:56,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:56,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:55:56,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:56,868 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:56,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:56,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:57,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777017006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:57,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:57,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:57,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777017006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:57,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:57,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:57,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:57,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:57,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:57,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T06:55:57,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:57,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:55:57,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:57,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:57,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:57,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:57,028 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T06:55:57,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/76189c5406404a54be4ec05e15e65f12 2024-11-28T06:55:57,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/98ea6c3fd015485c9fe06c565f2f716a is 50, key is test_row_0/B:col10/1732776956650/Put/seqid=0 2024-11-28T06:55:57,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742201_1377 (size=12001) 2024-11-28T06:55:57,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T06:55:57,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:57,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:55:57,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:57,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:57,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:57,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777017315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:57,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:57,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:57,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:57,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777017315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:57,324 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:57,324 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:57,324 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:57,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T06:55:57,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:57,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:55:57,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:57,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:57,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:57,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/98ea6c3fd015485c9fe06c565f2f716a 2024-11-28T06:55:57,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/39a6248931d44eb88a87112a45e147f5 is 50, key is test_row_0/C:col10/1732776956650/Put/seqid=0 2024-11-28T06:55:57,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T06:55:57,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:57,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:55:57,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:57,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:57,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:57,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:57,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742202_1378 (size=12001) 2024-11-28T06:55:57,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-28T06:55:57,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:57,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46398 deadline: 1732777017595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:57,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:57,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46420 deadline: 1732777017596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:57,607 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_0' in 00878c8327d194b967221c1933e24c9a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:55:57,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:57,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:57,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:57,607 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_1' in 00878c8327d194b967221c1933e24c9a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:55:57,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:57,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:57,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:57,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T06:55:57,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:57,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:55:57,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:57,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:57,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:57,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:57,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:57,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732777017645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:57,655 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_0' in 00878c8327d194b967221c1933e24c9a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:55:57,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:57,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:57,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:57,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T06:55:57,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:57,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:55:57,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:57,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:57,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:57,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:57,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:57,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777017836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:57,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:57,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777017836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:57,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:57,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:57,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:57,843 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:57,843 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:57,843 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:57,884 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/39a6248931d44eb88a87112a45e147f5 2024-11-28T06:55:57,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/76189c5406404a54be4ec05e15e65f12 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/76189c5406404a54be4ec05e15e65f12 2024-11-28T06:55:57,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/76189c5406404a54be4ec05e15e65f12, entries=200, sequenceid=48, filesize=14.0 K 2024-11-28T06:55:57,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/98ea6c3fd015485c9fe06c565f2f716a as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/98ea6c3fd015485c9fe06c565f2f716a 2024-11-28T06:55:57,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/98ea6c3fd015485c9fe06c565f2f716a, entries=150, sequenceid=48, filesize=11.7 K 2024-11-28T06:55:57,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/39a6248931d44eb88a87112a45e147f5 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/39a6248931d44eb88a87112a45e147f5 2024-11-28T06:55:57,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/39a6248931d44eb88a87112a45e147f5, entries=150, sequenceid=48, filesize=11.7 K 2024-11-28T06:55:57,897 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 00878c8327d194b967221c1933e24c9a in 1245ms, sequenceid=48, compaction requested=false 2024-11-28T06:55:57,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00878c8327d194b967221c1933e24c9a: 2024-11-28T06:55:57,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T06:55:57,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:57,940 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 00878c8327d194b967221c1933e24c9a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T06:55:57,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=A 2024-11-28T06:55:57,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:57,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=B 2024-11-28T06:55:57,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:57,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=C 2024-11-28T06:55:57,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:55:57,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/fcff6fdb6f93468cbfc15ddb2b7a4f57 is 50, key is test_row_1/A:col10/1732776956673/Put/seqid=0 2024-11-28T06:55:57,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742203_1379 (size=9657) 2024-11-28T06:55:58,346 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/fcff6fdb6f93468cbfc15ddb2b7a4f57 2024-11-28T06:55:58,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/1855db83f0874d589175d25c20fc8519 is 50, key is test_row_1/B:col10/1732776956673/Put/seqid=0 2024-11-28T06:55:58,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742204_1380 (size=9657) 2024-11-28T06:55:58,756 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/1855db83f0874d589175d25c20fc8519 2024-11-28T06:55:58,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/b3ffef4a980247dab4af259735378487 is 50, key is test_row_1/C:col10/1732776956673/Put/seqid=0 2024-11-28T06:55:58,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742205_1381 (size=9657) 2024-11-28T06:55:58,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 00878c8327d194b967221c1933e24c9a 2024-11-28T06:55:58,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:55:58,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732776958876, max timestamp=1732776958912Num uniques 859; 2024-11-28T06:55:58,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T06:55:58,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732776958876, max timestamp=1732776958912Num uniques 859; 2024-11-28T06:55:58,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T06:55:58,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732776958876, max timestamp=1732776958912Num uniques 859; 2024-11-28T06:55:58,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T06:55:58,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:58,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777018976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:58,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:58,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:58,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:58,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:58,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777018977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:58,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:58,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:58,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:59,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:59,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777019095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:59,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:59,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:59,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:59,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:59,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777019105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:59,114 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:59,114 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:59,114 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:59,175 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/b3ffef4a980247dab4af259735378487 2024-11-28T06:55:59,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/fcff6fdb6f93468cbfc15ddb2b7a4f57 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/fcff6fdb6f93468cbfc15ddb2b7a4f57 2024-11-28T06:55:59,182 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/fcff6fdb6f93468cbfc15ddb2b7a4f57, entries=100, sequenceid=58, filesize=9.4 K 2024-11-28T06:55:59,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/1855db83f0874d589175d25c20fc8519 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/1855db83f0874d589175d25c20fc8519 2024-11-28T06:55:59,186 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/1855db83f0874d589175d25c20fc8519, entries=100, sequenceid=58, filesize=9.4 K 2024-11-28T06:55:59,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/b3ffef4a980247dab4af259735378487 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/b3ffef4a980247dab4af259735378487 2024-11-28T06:55:59,190 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/b3ffef4a980247dab4af259735378487, entries=100, sequenceid=58, filesize=9.4 K 2024-11-28T06:55:59,190 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 00878c8327d194b967221c1933e24c9a in 1251ms, sequenceid=58, compaction requested=true 2024-11-28T06:55:59,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 00878c8327d194b967221c1933e24c9a: 2024-11-28T06:55:59,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:59,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-28T06:55:59,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-28T06:55:59,193 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-28T06:55:59,193 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.8520 sec 2024-11-28T06:55:59,194 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 3.8590 sec 2024-11-28T06:55:59,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 00878c8327d194b967221c1933e24c9a 2024-11-28T06:55:59,313 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00878c8327d194b967221c1933e24c9a 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-28T06:55:59,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=A 2024-11-28T06:55:59,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:55:59,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=B 2024-11-28T06:55:59,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:55:59,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=C 2024-11-28T06:55:59,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:55:59,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/bab4d46746024461a8f9f0a216338e5a is 50, key is test_row_0/A:col10/1732776958974/Put/seqid=0 2024-11-28T06:55:59,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742206_1382 (size=14341) 2024-11-28T06:55:59,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:59,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777019342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:59,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:59,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:59,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:59,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:59,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777019347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:59,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:59,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:59,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:59,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:59,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777019455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:59,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:59,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:59,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:59,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:59,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777019455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:59,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:59,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:59,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:59,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-28T06:55:59,496 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:55:59,497 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:55:59,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-28T06:55:59,498 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:55:59,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-28T06:55:59,499 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:55:59,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:55:59,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-28T06:55:59,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:59,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46420 deadline: 1732777019625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:59,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:59,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46398 deadline: 1732777019625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:59,630 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_1' in 00878c8327d194b967221c1933e24c9a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:55:59,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:59,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:59,630 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_0' in 00878c8327d194b967221c1933e24c9a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:55:59,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:59,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:59,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:59,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:59,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-28T06:55:59,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:59,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:55:59,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:59,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:59,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:59,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:59,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:59,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732777019676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:59,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:59,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777019676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:59,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:59,682 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:59,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777019676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:59,682 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:59,682 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_0' in 00878c8327d194b967221c1933e24c9a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:55:59,682 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:59,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:59,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:59,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:59,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:59,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:59,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:59,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/bab4d46746024461a8f9f0a216338e5a 2024-11-28T06:55:59,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/59cbc4ee8e594d5ea78b447c247340e5 is 50, key is test_row_0/B:col10/1732776958974/Put/seqid=0 2024-11-28T06:55:59,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742207_1383 (size=12001) 2024-11-28T06:55:59,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-28T06:55:59,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:59,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:55:59,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:59,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:59,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:59,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:59,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-28T06:55:59,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-28T06:55:59,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:59,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:55:59,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:55:59,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:59,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:59,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:55:59,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:59,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777019985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:59,991 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:59,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:55:59,991 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:59,991 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:55:59,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777019985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:55:59,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:59,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:55:59,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:00,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-28T06:56:00,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:00,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:00,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:00,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:00,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:00,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:00,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-28T06:56:00,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/59cbc4ee8e594d5ea78b447c247340e5 2024-11-28T06:56:00,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/e440df1e9784413ca3180d22cbd72235 is 50, key is test_row_0/C:col10/1732776958974/Put/seqid=0 2024-11-28T06:56:00,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742208_1384 (size=12001) 2024-11-28T06:56:00,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/e440df1e9784413ca3180d22cbd72235 2024-11-28T06:56:00,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/bab4d46746024461a8f9f0a216338e5a as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/bab4d46746024461a8f9f0a216338e5a 2024-11-28T06:56:00,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/bab4d46746024461a8f9f0a216338e5a, entries=200, sequenceid=95, filesize=14.0 K 2024-11-28T06:56:00,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/59cbc4ee8e594d5ea78b447c247340e5 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/59cbc4ee8e594d5ea78b447c247340e5 2024-11-28T06:56:00,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/59cbc4ee8e594d5ea78b447c247340e5, entries=150, sequenceid=95, filesize=11.7 K 2024-11-28T06:56:00,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/e440df1e9784413ca3180d22cbd72235 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/e440df1e9784413ca3180d22cbd72235 2024-11-28T06:56:00,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/e440df1e9784413ca3180d22cbd72235, entries=150, sequenceid=95, filesize=11.7 K 2024-11-28T06:56:00,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 00878c8327d194b967221c1933e24c9a in 851ms, sequenceid=95, compaction requested=true 2024-11-28T06:56:00,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00878c8327d194b967221c1933e24c9a: 2024-11-28T06:56:00,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00878c8327d194b967221c1933e24c9a:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:56:00,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:00,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00878c8327d194b967221c1933e24c9a:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:56:00,164 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T06:56:00,164 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T06:56:00,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:00,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00878c8327d194b967221c1933e24c9a:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:56:00,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:56:00,165 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T06:56:00,165 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50340 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T06:56:00,165 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): 00878c8327d194b967221c1933e24c9a/A is initiating minor compaction (all files) 2024-11-28T06:56:00,165 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): 00878c8327d194b967221c1933e24c9a/B is initiating minor compaction (all files) 2024-11-28T06:56:00,165 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00878c8327d194b967221c1933e24c9a/B in TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:00,165 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00878c8327d194b967221c1933e24c9a/A in TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:00,165 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/68c1bdb8b69b4971ba5fd8f93e44e5d1, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/98ea6c3fd015485c9fe06c565f2f716a, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/1855db83f0874d589175d25c20fc8519, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/59cbc4ee8e594d5ea78b447c247340e5] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp, totalSize=44.6 K 2024-11-28T06:56:00,165 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/446d92d35d744e32b42232548450e6ee, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/76189c5406404a54be4ec05e15e65f12, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/fcff6fdb6f93468cbfc15ddb2b7a4f57, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/bab4d46746024461a8f9f0a216338e5a] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp, totalSize=49.2 K 2024-11-28T06:56:00,165 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 446d92d35d744e32b42232548450e6ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732776955339 2024-11-28T06:56:00,165 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 68c1bdb8b69b4971ba5fd8f93e44e5d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732776955339 2024-11-28T06:56:00,166 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76189c5406404a54be4ec05e15e65f12, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732776955451 2024-11-28T06:56:00,166 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 98ea6c3fd015485c9fe06c565f2f716a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732776955451 2024-11-28T06:56:00,166 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 1855db83f0874d589175d25c20fc8519, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732776956673 2024-11-28T06:56:00,166 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcff6fdb6f93468cbfc15ddb2b7a4f57, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732776956673 2024-11-28T06:56:00,166 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting bab4d46746024461a8f9f0a216338e5a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732776958956 2024-11-28T06:56:00,166 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 59cbc4ee8e594d5ea78b447c247340e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732776958956 2024-11-28T06:56:00,174 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00878c8327d194b967221c1933e24c9a#A#compaction#319 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:00,175 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/1cbc755a7da048b99564930c168df5e4 is 50, key is test_row_0/A:col10/1732776958974/Put/seqid=0 2024-11-28T06:56:00,178 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00878c8327d194b967221c1933e24c9a#B#compaction#320 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:00,178 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/67c202edc110459da488db5896fd8362 is 50, key is test_row_0/B:col10/1732776958974/Put/seqid=0 2024-11-28T06:56:00,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742209_1385 (size=12139) 2024-11-28T06:56:00,187 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/1cbc755a7da048b99564930c168df5e4 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/1cbc755a7da048b99564930c168df5e4 2024-11-28T06:56:00,191 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 00878c8327d194b967221c1933e24c9a/A of 00878c8327d194b967221c1933e24c9a into 1cbc755a7da048b99564930c168df5e4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:00,191 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00878c8327d194b967221c1933e24c9a: 2024-11-28T06:56:00,191 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., storeName=00878c8327d194b967221c1933e24c9a/A, priority=12, startTime=1732776960164; duration=0sec 2024-11-28T06:56:00,191 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:56:00,191 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00878c8327d194b967221c1933e24c9a:A 2024-11-28T06:56:00,191 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T06:56:00,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742210_1386 (size=12139) 2024-11-28T06:56:00,193 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T06:56:00,193 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): 00878c8327d194b967221c1933e24c9a/C is initiating minor compaction (all files) 2024-11-28T06:56:00,193 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00878c8327d194b967221c1933e24c9a/C in TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:00,193 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/4f54c3c115164b24869330622a0d3412, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/39a6248931d44eb88a87112a45e147f5, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/b3ffef4a980247dab4af259735378487, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/e440df1e9784413ca3180d22cbd72235] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp, totalSize=44.6 K 2024-11-28T06:56:00,194 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f54c3c115164b24869330622a0d3412, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732776955339 2024-11-28T06:56:00,196 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39a6248931d44eb88a87112a45e147f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732776955451 2024-11-28T06:56:00,197 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3ffef4a980247dab4af259735378487, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732776956673 2024-11-28T06:56:00,197 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting e440df1e9784413ca3180d22cbd72235, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732776958956 2024-11-28T06:56:00,203 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/67c202edc110459da488db5896fd8362 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/67c202edc110459da488db5896fd8362 2024-11-28T06:56:00,211 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 00878c8327d194b967221c1933e24c9a/B of 00878c8327d194b967221c1933e24c9a into 67c202edc110459da488db5896fd8362(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:00,211 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00878c8327d194b967221c1933e24c9a: 2024-11-28T06:56:00,211 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., storeName=00878c8327d194b967221c1933e24c9a/B, priority=12, startTime=1732776960164; duration=0sec 2024-11-28T06:56:00,211 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:00,212 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00878c8327d194b967221c1933e24c9a:B 2024-11-28T06:56:00,219 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00878c8327d194b967221c1933e24c9a#C#compaction#321 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:00,219 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/323df53dc3b849f0ac6ce03b6475f3a8 is 50, key is test_row_0/C:col10/1732776958974/Put/seqid=0 2024-11-28T06:56:00,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742211_1387 (size=12139) 2024-11-28T06:56:00,253 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/323df53dc3b849f0ac6ce03b6475f3a8 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/323df53dc3b849f0ac6ce03b6475f3a8 2024-11-28T06:56:00,258 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 00878c8327d194b967221c1933e24c9a/C of 00878c8327d194b967221c1933e24c9a into 323df53dc3b849f0ac6ce03b6475f3a8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:00,258 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00878c8327d194b967221c1933e24c9a: 2024-11-28T06:56:00,258 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., storeName=00878c8327d194b967221c1933e24c9a/C, priority=12, startTime=1732776960164; duration=0sec 2024-11-28T06:56:00,258 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:00,258 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00878c8327d194b967221c1933e24c9a:C 2024-11-28T06:56:00,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-28T06:56:00,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:00,264 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 00878c8327d194b967221c1933e24c9a 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-28T06:56:00,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=A 2024-11-28T06:56:00,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:00,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=B 2024-11-28T06:56:00,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:00,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=C 2024-11-28T06:56:00,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:00,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/c33819b0c79543eb827db5d66f79ef61 is 50, key is test_row_0/A:col10/1732776959341/Put/seqid=0 2024-11-28T06:56:00,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742212_1388 (size=12001) 2024-11-28T06:56:00,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:00,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 00878c8327d194b967221c1933e24c9a 2024-11-28T06:56:00,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732776960527, max timestamp=1732776960555Num uniques 859; 2024-11-28T06:56:00,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732776960527, max timestamp=1732776960555Num uniques 859; 2024-11-28T06:56:00,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732776960527, max timestamp=1732776960555Num uniques 859; 2024-11-28T06:56:00,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T06:56:00,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T06:56:00,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T06:56:00,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:00,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777020621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:00,627 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:00,627 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:00,627 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:00,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:00,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777020624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:00,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:00,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:00,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:00,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-28T06:56:00,675 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/c33819b0c79543eb827db5d66f79ef61 2024-11-28T06:56:00,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/aabbc387dcbb4fc9bd9301dd5f1cf86d is 50, key is test_row_0/B:col10/1732776959341/Put/seqid=0 2024-11-28T06:56:00,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742213_1389 (size=12001) 2024-11-28T06:56:00,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:00,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777020735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:00,746 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:00,746 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:00,746 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:00,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:00,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777020746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:00,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:00,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:00,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:00,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:00,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777020955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:00,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:00,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:00,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:00,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:00,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777020955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:00,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:00,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:00,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:01,084 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/aabbc387dcbb4fc9bd9301dd5f1cf86d 2024-11-28T06:56:01,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/b731bfe1e147444a80c11af3a3e2cebd is 50, key is test_row_0/C:col10/1732776959341/Put/seqid=0 2024-11-28T06:56:01,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742214_1390 (size=12001) 2024-11-28T06:56:01,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:01,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777021276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:01,283 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:01,283 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:01,283 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:01,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:01,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777021276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:01,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:01,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:01,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:01,519 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/b731bfe1e147444a80c11af3a3e2cebd 2024-11-28T06:56:01,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/c33819b0c79543eb827db5d66f79ef61 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/c33819b0c79543eb827db5d66f79ef61 2024-11-28T06:56:01,525 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/c33819b0c79543eb827db5d66f79ef61, entries=150, sequenceid=107, filesize=11.7 K 2024-11-28T06:56:01,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/aabbc387dcbb4fc9bd9301dd5f1cf86d as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/aabbc387dcbb4fc9bd9301dd5f1cf86d 2024-11-28T06:56:01,529 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/aabbc387dcbb4fc9bd9301dd5f1cf86d, entries=150, sequenceid=107, filesize=11.7 K 2024-11-28T06:56:01,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/b731bfe1e147444a80c11af3a3e2cebd as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/b731bfe1e147444a80c11af3a3e2cebd 2024-11-28T06:56:01,533 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/b731bfe1e147444a80c11af3a3e2cebd, entries=150, sequenceid=107, filesize=11.7 K 2024-11-28T06:56:01,534 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 00878c8327d194b967221c1933e24c9a in 1270ms, sequenceid=107, compaction requested=false 2024-11-28T06:56:01,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 00878c8327d194b967221c1933e24c9a: 2024-11-28T06:56:01,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:01,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-28T06:56:01,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-28T06:56:01,536 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-28T06:56:01,536 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0360 sec 2024-11-28T06:56:01,538 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 2.0400 sec 2024-11-28T06:56:01,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-28T06:56:01,646 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:56:01,646 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:56:01,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-28T06:56:01,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-28T06:56:01,648 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:56:01,649 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:56:01,649 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:56:01,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-28T06:56:01,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-28T06:56:01,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:01,801 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 00878c8327d194b967221c1933e24c9a 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-28T06:56:01,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=A 2024-11-28T06:56:01,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:56:01,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=B 2024-11-28T06:56:01,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:56:01,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=C 2024-11-28T06:56:01,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:56:01,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 00878c8327d194b967221c1933e24c9a 2024-11-28T06:56:01,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:01,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/9bd92a9edcbb450aa02d07fec9c73713 is 50, key is test_row_0/A:col10/1732776960585/Put/seqid=0 2024-11-28T06:56:01,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742215_1391 (size=12151) 2024-11-28T06:56:01,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:01,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777021817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:01,827 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:01,827 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:01,827 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:01,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:01,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777021820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:01,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:01,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777021935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:01,939 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:01,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:01,939 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:01,939 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:01,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777021935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:01,939 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:01,939 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:01,939 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:01,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-28T06:56:02,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:02,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777022145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:02,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:02,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777022145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:02,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:02,209 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/9bd92a9edcbb450aa02d07fec9c73713 2024-11-28T06:56:02,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/3b5cb3480dda4d3fae315fea4de1f6f6 is 50, key is test_row_0/B:col10/1732776960585/Put/seqid=0 2024-11-28T06:56:02,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742216_1392 (size=12151) 2024-11-28T06:56:02,220 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/3b5cb3480dda4d3fae315fea4de1f6f6 2024-11-28T06:56:02,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/0f74764310254e478a603f468db4cb1d is 50, key is test_row_0/C:col10/1732776960585/Put/seqid=0 2024-11-28T06:56:02,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742217_1393 (size=12151) 2024-11-28T06:56:02,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-28T06:56:02,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:02,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777022456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:02,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:02,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:02,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:02,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:02,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777022456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:02,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:02,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:02,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:02,630 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/0f74764310254e478a603f468db4cb1d 2024-11-28T06:56:02,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/9bd92a9edcbb450aa02d07fec9c73713 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/9bd92a9edcbb450aa02d07fec9c73713 2024-11-28T06:56:02,636 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/9bd92a9edcbb450aa02d07fec9c73713, entries=150, sequenceid=144, filesize=11.9 K 2024-11-28T06:56:02,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/3b5cb3480dda4d3fae315fea4de1f6f6 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/3b5cb3480dda4d3fae315fea4de1f6f6 2024-11-28T06:56:02,639 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/3b5cb3480dda4d3fae315fea4de1f6f6, entries=150, sequenceid=144, filesize=11.9 K 2024-11-28T06:56:02,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/0f74764310254e478a603f468db4cb1d as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/0f74764310254e478a603f468db4cb1d 2024-11-28T06:56:02,643 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/0f74764310254e478a603f468db4cb1d, entries=150, sequenceid=144, filesize=11.9 K 2024-11-28T06:56:02,644 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 00878c8327d194b967221c1933e24c9a in 843ms, sequenceid=144, compaction requested=true 2024-11-28T06:56:02,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 00878c8327d194b967221c1933e24c9a: 2024-11-28T06:56:02,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:02,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-28T06:56:02,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-28T06:56:02,647 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-28T06:56:02,647 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 996 msec 2024-11-28T06:56:02,648 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.0010 sec 2024-11-28T06:56:02,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-28T06:56:02,785 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:56:02,786 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:56:02,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-28T06:56:02,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-28T06:56:02,787 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:56:02,788 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:56:02,788 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:56:02,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-28T06:56:02,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-28T06:56:02,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:02,940 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 00878c8327d194b967221c1933e24c9a 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-28T06:56:02,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=A 2024-11-28T06:56:02,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:02,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=B 2024-11-28T06:56:02,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:02,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=C 2024-11-28T06:56:02,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:02,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/2b5894fd9b9d4a9a9ad3f0c51ab30296 is 50, key is test_row_0/A:col10/1732776961814/Put/seqid=0 2024-11-28T06:56:02,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742218_1394 (size=12151) 2024-11-28T06:56:02,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 00878c8327d194b967221c1933e24c9a 2024-11-28T06:56:02,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:03,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732776963009, max timestamp=1732776963063Num uniques 859; 2024-11-28T06:56:03,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T06:56:03,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732776963009, max timestamp=1732776963063Num uniques 859; 2024-11-28T06:56:03,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T06:56:03,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732776963009, max timestamp=1732776963063Num uniques 859; 2024-11-28T06:56:03,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T06:56:03,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-28T06:56:03,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732776963128, max timestamp=1732776963147Num uniques 1018; 2024-11-28T06:56:03,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732776963128, max timestamp=1732776963147Num uniques 1018; 2024-11-28T06:56:03,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-28T06:56:03,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732776963128, max timestamp=1732776963147Num uniques 1018; 2024-11-28T06:56:03,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-28T06:56:03,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-28T06:56:03,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:03,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777023220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:03,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:03,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:03,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:03,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:03,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777023223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:03,235 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:03,235 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:03,235 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:03,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:03,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777023335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:03,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:03,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:03,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:03,350 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/2b5894fd9b9d4a9a9ad3f0c51ab30296 2024-11-28T06:56:03,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:03,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777023345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:03,353 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:03,353 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:03,353 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:03,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/bb690150437946cea82749f4a836271c is 50, key is test_row_0/B:col10/1732776961814/Put/seqid=0 2024-11-28T06:56:03,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742219_1395 (size=12151) 2024-11-28T06:56:03,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-28T06:56:03,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:03,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777023555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:03,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:03,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:03,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:03,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777023555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:03,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:03,566 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:03,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:03,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:03,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:03,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46398 deadline: 1732777023635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:03,644 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_0' in 00878c8327d194b967221c1933e24c9a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:56:03,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:03,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:03,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:03,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:03,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46420 deadline: 1732777023665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:03,672 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_1' in 00878c8327d194b967221c1933e24c9a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8259 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:56:03,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:03,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:03,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:03,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:03,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732777023705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:03,715 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_0' in 00878c8327d194b967221c1933e24c9a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8249 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:56:03,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:03,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:03,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:03,759 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/bb690150437946cea82749f4a836271c 2024-11-28T06:56:03,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/83a107ac4d7940c5af3f4906e4674b96 is 50, key is test_row_0/C:col10/1732776961814/Put/seqid=0 2024-11-28T06:56:03,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742220_1396 (size=12151) 2024-11-28T06:56:03,769 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/83a107ac4d7940c5af3f4906e4674b96 2024-11-28T06:56:03,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/2b5894fd9b9d4a9a9ad3f0c51ab30296 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/2b5894fd9b9d4a9a9ad3f0c51ab30296 2024-11-28T06:56:03,776 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/2b5894fd9b9d4a9a9ad3f0c51ab30296, entries=150, sequenceid=153, filesize=11.9 K 2024-11-28T06:56:03,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/bb690150437946cea82749f4a836271c as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/bb690150437946cea82749f4a836271c 2024-11-28T06:56:03,779 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/bb690150437946cea82749f4a836271c, entries=150, sequenceid=153, filesize=11.9 K 2024-11-28T06:56:03,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/83a107ac4d7940c5af3f4906e4674b96 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/83a107ac4d7940c5af3f4906e4674b96 2024-11-28T06:56:03,782 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/83a107ac4d7940c5af3f4906e4674b96, entries=150, sequenceid=153, filesize=11.9 K 2024-11-28T06:56:03,783 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 00878c8327d194b967221c1933e24c9a in 843ms, sequenceid=153, compaction requested=true 2024-11-28T06:56:03,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 00878c8327d194b967221c1933e24c9a: 2024-11-28T06:56:03,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:03,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-28T06:56:03,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-28T06:56:03,786 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-28T06:56:03,786 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 996 msec 2024-11-28T06:56:03,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.0000 sec 2024-11-28T06:56:03,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 00878c8327d194b967221c1933e24c9a 2024-11-28T06:56:03,892 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00878c8327d194b967221c1933e24c9a 3/3 column families, dataSize=147.60 KB heapSize=420 KB 2024-11-28T06:56:03,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732776963192, max timestamp=1732776963888Num uniques 1077; 2024-11-28T06:56:03,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732776963192, max timestamp=1732776963888Num uniques 1077; 2024-11-28T06:56:03,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-28T06:56:03,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-28T06:56:03,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=A 2024-11-28T06:56:03,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:56:03,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=B 2024-11-28T06:56:03,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:56:03,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=C 2024-11-28T06:56:03,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732776963192, max timestamp=1732776963888Num uniques 1077; 2024-11-28T06:56:03,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:56:03,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-28T06:56:03,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/74c84ab976134dd88e28ccccf64142c8 is 50, key is test_row_0/A:col10/1732776963890/Put/seqid=0 2024-11-28T06:56:03,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742221_1397 (size=14541) 2024-11-28T06:56:03,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-28T06:56:03,925 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:56:03,925 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:56:03,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-28T06:56:03,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-28T06:56:03,927 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:56:03,927 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:56:03,927 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:56:03,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732776963930, max timestamp=1732776963988Num uniques 859; 2024-11-28T06:56:03,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732776963930, max timestamp=1732776963988Num uniques 859; 2024-11-28T06:56:03,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T06:56:03,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T06:56:04,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-28T06:56:04,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-28T06:56:04,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:04,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:04,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:04,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:04,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:04,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:04,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:04,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777024074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:04,082 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:04,082 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:04,082 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:04,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:04,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777024081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:04,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:04,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:04,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:04,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:04,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777024185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:04,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:04,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:04,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:04,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:04,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777024195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:04,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:04,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:04,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:04,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-28T06:56:04,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:04,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:04,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:04,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:04,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:04,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:04,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-28T06:56:04,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/74c84ab976134dd88e28ccccf64142c8 2024-11-28T06:56:04,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/7d133f98a67d488694560af4874f4957 is 50, key is test_row_0/B:col10/1732776963890/Put/seqid=0 2024-11-28T06:56:04,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742222_1398 (size=12151) 2024-11-28T06:56:04,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-28T06:56:04,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:04,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:04,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:04,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:04,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:04,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:04,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:04,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777024405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:04,414 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:04,414 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:04,414 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:04,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:04,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777024426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:04,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:04,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:04,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:04,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-28T06:56:04,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:04,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:04,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:04,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:04,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:04,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:04,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-28T06:56:04,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-28T06:56:04,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:04,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:04,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:04,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:04,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:04,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:04,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/7d133f98a67d488694560af4874f4957 2024-11-28T06:56:04,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/8ff90d93149a40e6801d3ecdf17e7621 is 50, key is test_row_0/C:col10/1732776963890/Put/seqid=0 2024-11-28T06:56:04,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742223_1399 (size=12151) 2024-11-28T06:56:04,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:04,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777024715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:04,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:04,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:04,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:04,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:04,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777024745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:04,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:04,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:04,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:04,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-28T06:56:04,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:04,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:04,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:04,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:04,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:04,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:04,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-28T06:56:04,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:04,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:04,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:04,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:04,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:04,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:05,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-28T06:56:05,120 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/8ff90d93149a40e6801d3ecdf17e7621 2024-11-28T06:56:05,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/74c84ab976134dd88e28ccccf64142c8 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/74c84ab976134dd88e28ccccf64142c8 2024-11-28T06:56:05,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/74c84ab976134dd88e28ccccf64142c8, entries=200, sequenceid=209, filesize=14.2 K 2024-11-28T06:56:05,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/7d133f98a67d488694560af4874f4957 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/7d133f98a67d488694560af4874f4957 2024-11-28T06:56:05,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/7d133f98a67d488694560af4874f4957, entries=150, sequenceid=209, filesize=11.9 K 2024-11-28T06:56:05,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/8ff90d93149a40e6801d3ecdf17e7621 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/8ff90d93149a40e6801d3ecdf17e7621 2024-11-28T06:56:05,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/8ff90d93149a40e6801d3ecdf17e7621, entries=150, sequenceid=209, filesize=11.9 K 2024-11-28T06:56:05,132 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~84.42 KB/86448, currentSize=149.83 KB/153430 for 00878c8327d194b967221c1933e24c9a in 1240ms, sequenceid=209, compaction requested=true 2024-11-28T06:56:05,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00878c8327d194b967221c1933e24c9a: 2024-11-28T06:56:05,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00878c8327d194b967221c1933e24c9a:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:56:05,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:05,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00878c8327d194b967221c1933e24c9a:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:56:05,132 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-28T06:56:05,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:05,132 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-28T06:56:05,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00878c8327d194b967221c1933e24c9a:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:56:05,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:56:05,133 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60593 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-28T06:56:05,133 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62983 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-28T06:56:05,133 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): 00878c8327d194b967221c1933e24c9a/B is initiating minor compaction (all files) 2024-11-28T06:56:05,133 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): 00878c8327d194b967221c1933e24c9a/A is initiating minor compaction (all files) 2024-11-28T06:56:05,133 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00878c8327d194b967221c1933e24c9a/B in TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:05,133 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00878c8327d194b967221c1933e24c9a/A in TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:05,133 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/67c202edc110459da488db5896fd8362, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/aabbc387dcbb4fc9bd9301dd5f1cf86d, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/3b5cb3480dda4d3fae315fea4de1f6f6, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/bb690150437946cea82749f4a836271c, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/7d133f98a67d488694560af4874f4957] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp, totalSize=59.2 K 2024-11-28T06:56:05,134 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/1cbc755a7da048b99564930c168df5e4, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/c33819b0c79543eb827db5d66f79ef61, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/9bd92a9edcbb450aa02d07fec9c73713, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/2b5894fd9b9d4a9a9ad3f0c51ab30296, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/74c84ab976134dd88e28ccccf64142c8] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp, totalSize=61.5 K 2024-11-28T06:56:05,134 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 67c202edc110459da488db5896fd8362, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732776958956 2024-11-28T06:56:05,134 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cbc755a7da048b99564930c168df5e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732776958956 2024-11-28T06:56:05,134 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting aabbc387dcbb4fc9bd9301dd5f1cf86d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732776959322 2024-11-28T06:56:05,134 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting c33819b0c79543eb827db5d66f79ef61, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732776959322 2024-11-28T06:56:05,135 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b5cb3480dda4d3fae315fea4de1f6f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732776960585 2024-11-28T06:56:05,135 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bd92a9edcbb450aa02d07fec9c73713, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732776960585 2024-11-28T06:56:05,135 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting bb690150437946cea82749f4a836271c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732776961808 2024-11-28T06:56:05,135 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b5894fd9b9d4a9a9ad3f0c51ab30296, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732776961808 2024-11-28T06:56:05,135 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d133f98a67d488694560af4874f4957, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732776963222 2024-11-28T06:56:05,135 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74c84ab976134dd88e28ccccf64142c8, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732776963219 2024-11-28T06:56:05,146 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00878c8327d194b967221c1933e24c9a#B#compaction#334 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:05,146 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/f3159ebb288a4f1882cbd123e039ecbe is 50, key is test_row_0/B:col10/1732776963890/Put/seqid=0 2024-11-28T06:56:05,147 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00878c8327d194b967221c1933e24c9a#A#compaction#335 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:05,147 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/89592cc4dd344b38a138308b70e144d4 is 50, key is test_row_0/A:col10/1732776963890/Put/seqid=0 2024-11-28T06:56:05,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742224_1400 (size=12459) 2024-11-28T06:56:05,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-28T06:56:05,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:05,150 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 00878c8327d194b967221c1933e24c9a 3/3 column families, dataSize=149.83 KB heapSize=433.31 KB 2024-11-28T06:56:05,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=A 2024-11-28T06:56:05,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:56:05,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=B 2024-11-28T06:56:05,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:56:05,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=C 2024-11-28T06:56:05,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:56:05,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742225_1401 (size=12459) 2024-11-28T06:56:05,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/25fd457ddde540fdac27db87629c3a07 is 50, key is test_row_0/A:col10/1732776964079/Put/seqid=0 2024-11-28T06:56:05,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742226_1402 (size=12151) 2024-11-28T06:56:05,158 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=69.33 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/25fd457ddde540fdac27db87629c3a07 2024-11-28T06:56:05,159 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/89592cc4dd344b38a138308b70e144d4 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/89592cc4dd344b38a138308b70e144d4 2024-11-28T06:56:05,163 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 00878c8327d194b967221c1933e24c9a/A of 00878c8327d194b967221c1933e24c9a into 89592cc4dd344b38a138308b70e144d4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:05,163 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00878c8327d194b967221c1933e24c9a: 2024-11-28T06:56:05,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/72916856ee184a7aa8759edaf856ccf3 is 50, key is test_row_0/B:col10/1732776964079/Put/seqid=0 2024-11-28T06:56:05,164 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., storeName=00878c8327d194b967221c1933e24c9a/A, priority=11, startTime=1732776965132; duration=0sec 2024-11-28T06:56:05,164 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:56:05,164 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00878c8327d194b967221c1933e24c9a:A 2024-11-28T06:56:05,164 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-28T06:56:05,165 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60593 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-28T06:56:05,165 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): 00878c8327d194b967221c1933e24c9a/C is initiating minor compaction (all files) 2024-11-28T06:56:05,165 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00878c8327d194b967221c1933e24c9a/C in TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:05,166 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/323df53dc3b849f0ac6ce03b6475f3a8, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/b731bfe1e147444a80c11af3a3e2cebd, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/0f74764310254e478a603f468db4cb1d, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/83a107ac4d7940c5af3f4906e4674b96, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/8ff90d93149a40e6801d3ecdf17e7621] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp, totalSize=59.2 K 2024-11-28T06:56:05,166 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 323df53dc3b849f0ac6ce03b6475f3a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732776958956 2024-11-28T06:56:05,166 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting b731bfe1e147444a80c11af3a3e2cebd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732776959322 2024-11-28T06:56:05,166 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f74764310254e478a603f468db4cb1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732776960585 2024-11-28T06:56:05,166 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83a107ac4d7940c5af3f4906e4674b96, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732776961808 2024-11-28T06:56:05,167 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ff90d93149a40e6801d3ecdf17e7621, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732776963222 2024-11-28T06:56:05,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742227_1403 (size=12151) 2024-11-28T06:56:05,169 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/72916856ee184a7aa8759edaf856ccf3 2024-11-28T06:56:05,174 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00878c8327d194b967221c1933e24c9a#C#compaction#338 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:05,175 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/4663aebb4c4f4d9db6f69e641bb1b486 is 50, key is test_row_0/C:col10/1732776963890/Put/seqid=0 2024-11-28T06:56:05,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/b2fe3d0082f443389b26ad75ab71d200 is 50, key is test_row_0/C:col10/1732776964079/Put/seqid=0 2024-11-28T06:56:05,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742229_1405 (size=12151) 2024-11-28T06:56:05,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742228_1404 (size=12459) 2024-11-28T06:56:05,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 00878c8327d194b967221c1933e24c9a 2024-11-28T06:56:05,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:05,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:05,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777025265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:05,282 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:05,282 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:05,282 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:05,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:05,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777025282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:05,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:05,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:05,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:05,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:05,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777025385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:05,395 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:05,395 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:05,395 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:05,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:05,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777025396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:05,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:05,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:05,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:05,553 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/f3159ebb288a4f1882cbd123e039ecbe as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/f3159ebb288a4f1882cbd123e039ecbe 2024-11-28T06:56:05,556 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 00878c8327d194b967221c1933e24c9a/B of 00878c8327d194b967221c1933e24c9a into f3159ebb288a4f1882cbd123e039ecbe(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:05,556 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00878c8327d194b967221c1933e24c9a: 2024-11-28T06:56:05,556 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., storeName=00878c8327d194b967221c1933e24c9a/B, priority=11, startTime=1732776965132; duration=0sec 2024-11-28T06:56:05,557 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:05,557 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00878c8327d194b967221c1933e24c9a:B 2024-11-28T06:56:05,579 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/b2fe3d0082f443389b26ad75ab71d200 2024-11-28T06:56:05,582 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/4663aebb4c4f4d9db6f69e641bb1b486 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/4663aebb4c4f4d9db6f69e641bb1b486 2024-11-28T06:56:05,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/25fd457ddde540fdac27db87629c3a07 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/25fd457ddde540fdac27db87629c3a07 2024-11-28T06:56:05,587 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/25fd457ddde540fdac27db87629c3a07, entries=150, sequenceid=243, filesize=11.9 K 2024-11-28T06:56:05,587 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 00878c8327d194b967221c1933e24c9a/C of 00878c8327d194b967221c1933e24c9a into 4663aebb4c4f4d9db6f69e641bb1b486(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:05,587 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00878c8327d194b967221c1933e24c9a: 2024-11-28T06:56:05,587 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., storeName=00878c8327d194b967221c1933e24c9a/C, priority=11, startTime=1732776965132; duration=0sec 2024-11-28T06:56:05,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/72916856ee184a7aa8759edaf856ccf3 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/72916856ee184a7aa8759edaf856ccf3 2024-11-28T06:56:05,587 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:05,587 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00878c8327d194b967221c1933e24c9a:C 2024-11-28T06:56:05,590 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/72916856ee184a7aa8759edaf856ccf3, entries=150, sequenceid=243, filesize=11.9 K 2024-11-28T06:56:05,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/b2fe3d0082f443389b26ad75ab71d200 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/b2fe3d0082f443389b26ad75ab71d200 2024-11-28T06:56:05,593 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/b2fe3d0082f443389b26ad75ab71d200, entries=150, sequenceid=243, filesize=11.9 K 2024-11-28T06:56:05,594 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~149.83 KB/153430, heapSize ~433.27 KB/443664, currentSize=26.84 KB/27480 for 00878c8327d194b967221c1933e24c9a in 444ms, sequenceid=243, compaction requested=false 2024-11-28T06:56:05,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 00878c8327d194b967221c1933e24c9a: 2024-11-28T06:56:05,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:05,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-28T06:56:05,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-28T06:56:05,596 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-28T06:56:05,596 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6680 sec 2024-11-28T06:56:05,598 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.6720 sec 2024-11-28T06:56:05,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 00878c8327d194b967221c1933e24c9a 2024-11-28T06:56:05,609 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00878c8327d194b967221c1933e24c9a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T06:56:05,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=A 2024-11-28T06:56:05,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:05,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=B 2024-11-28T06:56:05,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:05,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=C 2024-11-28T06:56:05,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:05,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/2e436f5c37f847aabfa527bfcaff4107 is 50, key is test_row_0/A:col10/1732776965608/Put/seqid=0 2024-11-28T06:56:05,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742230_1406 (size=16931) 2024-11-28T06:56:05,617 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/2e436f5c37f847aabfa527bfcaff4107 2024-11-28T06:56:05,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/fc57924001ba47278e73aef789c8fa05 is 50, key is test_row_0/B:col10/1732776965608/Put/seqid=0 2024-11-28T06:56:05,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742231_1407 (size=12151) 2024-11-28T06:56:05,687 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732776965662, max timestamp=1732776965684Num uniques 859; 2024-11-28T06:56:05,687 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732776965662, max timestamp=1732776965684Num uniques 859; 2024-11-28T06:56:05,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732776965662, max timestamp=1732776965684Num uniques 859; 2024-11-28T06:56:05,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T06:56:05,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T06:56:05,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T06:56:05,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:05,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777025754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:05,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:05,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777025765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:05,768 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:05,768 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:05,768 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:05,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:05,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777025875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:05,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:05,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:05,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:05,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:05,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777025876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:05,888 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:05,888 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:05,888 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:06,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/fc57924001ba47278e73aef789c8fa05 2024-11-28T06:56:06,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/a386b050049f4bce8586bfca5b3e989e is 50, key is test_row_0/C:col10/1732776965608/Put/seqid=0 2024-11-28T06:56:06,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742232_1408 (size=12151) 2024-11-28T06:56:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-28T06:56:06,075 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:56:06,076 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:56:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-28T06:56:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-28T06:56:06,077 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:56:06,078 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:56:06,078 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:56:06,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777026095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:06,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:06,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777026095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:06,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:06,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:06,102 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:06,102 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:06,102 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-28T06:56:06,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-28T06:56:06,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:06,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:06,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:06,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:06,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:06,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-28T06:56:06,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:06,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:06,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:06,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:06,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-28T06:56:06,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:06,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777026415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:06,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:06,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:06,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:06,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:06,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777026415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:06,422 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:06,422 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:06,422 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:06,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/a386b050049f4bce8586bfca5b3e989e 2024-11-28T06:56:06,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/2e436f5c37f847aabfa527bfcaff4107 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/2e436f5c37f847aabfa527bfcaff4107 2024-11-28T06:56:06,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/2e436f5c37f847aabfa527bfcaff4107, entries=250, sequenceid=256, filesize=16.5 K 2024-11-28T06:56:06,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/fc57924001ba47278e73aef789c8fa05 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/fc57924001ba47278e73aef789c8fa05 2024-11-28T06:56:06,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/fc57924001ba47278e73aef789c8fa05, entries=150, sequenceid=256, filesize=11.9 K 2024-11-28T06:56:06,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/a386b050049f4bce8586bfca5b3e989e as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/a386b050049f4bce8586bfca5b3e989e 2024-11-28T06:56:06,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/a386b050049f4bce8586bfca5b3e989e, entries=150, sequenceid=256, filesize=11.9 K 2024-11-28T06:56:06,452 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 00878c8327d194b967221c1933e24c9a in 843ms, sequenceid=256, compaction requested=true 2024-11-28T06:56:06,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00878c8327d194b967221c1933e24c9a: 2024-11-28T06:56:06,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00878c8327d194b967221c1933e24c9a:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:56:06,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:06,452 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:56:06,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00878c8327d194b967221c1933e24c9a:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:56:06,452 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:56:06,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:06,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00878c8327d194b967221c1933e24c9a:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:56:06,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:56:06,453 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41541 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:56:06,453 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): 00878c8327d194b967221c1933e24c9a/A is initiating minor compaction (all files) 2024-11-28T06:56:06,453 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:56:06,453 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): 00878c8327d194b967221c1933e24c9a/B is initiating minor compaction (all files) 2024-11-28T06:56:06,453 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00878c8327d194b967221c1933e24c9a/A in TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:06,453 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00878c8327d194b967221c1933e24c9a/B in TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:06,453 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/89592cc4dd344b38a138308b70e144d4, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/25fd457ddde540fdac27db87629c3a07, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/2e436f5c37f847aabfa527bfcaff4107] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp, totalSize=40.6 K 2024-11-28T06:56:06,453 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/f3159ebb288a4f1882cbd123e039ecbe, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/72916856ee184a7aa8759edaf856ccf3, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/fc57924001ba47278e73aef789c8fa05] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp, totalSize=35.9 K 2024-11-28T06:56:06,453 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting f3159ebb288a4f1882cbd123e039ecbe, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732776963222 2024-11-28T06:56:06,453 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89592cc4dd344b38a138308b70e144d4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732776963222 2024-11-28T06:56:06,453 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 72916856ee184a7aa8759edaf856ccf3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732776964025 2024-11-28T06:56:06,454 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25fd457ddde540fdac27db87629c3a07, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732776964025 2024-11-28T06:56:06,454 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting fc57924001ba47278e73aef789c8fa05, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732776965281 2024-11-28T06:56:06,454 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e436f5c37f847aabfa527bfcaff4107, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732776965264 2024-11-28T06:56:06,461 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00878c8327d194b967221c1933e24c9a#A#compaction#343 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:06,461 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/882bace7a22944cc94656ef870bc9876 is 50, key is test_row_0/A:col10/1732776965608/Put/seqid=0 2024-11-28T06:56:06,463 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00878c8327d194b967221c1933e24c9a#B#compaction#344 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:06,464 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/ea0fc35b687e409294a2ca5d4c66fd06 is 50, key is test_row_0/B:col10/1732776965608/Put/seqid=0 2024-11-28T06:56:06,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742233_1409 (size=12561) 2024-11-28T06:56:06,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742234_1410 (size=12561) 2024-11-28T06:56:06,473 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/ea0fc35b687e409294a2ca5d4c66fd06 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/ea0fc35b687e409294a2ca5d4c66fd06 2024-11-28T06:56:06,477 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00878c8327d194b967221c1933e24c9a/B of 00878c8327d194b967221c1933e24c9a into ea0fc35b687e409294a2ca5d4c66fd06(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:06,477 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00878c8327d194b967221c1933e24c9a: 2024-11-28T06:56:06,477 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., storeName=00878c8327d194b967221c1933e24c9a/B, priority=13, startTime=1732776966452; duration=0sec 2024-11-28T06:56:06,477 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:56:06,477 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00878c8327d194b967221c1933e24c9a:B 2024-11-28T06:56:06,477 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:56:06,478 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:56:06,478 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): 00878c8327d194b967221c1933e24c9a/C is initiating minor compaction (all files) 2024-11-28T06:56:06,478 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00878c8327d194b967221c1933e24c9a/C in TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:06,478 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/4663aebb4c4f4d9db6f69e641bb1b486, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/b2fe3d0082f443389b26ad75ab71d200, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/a386b050049f4bce8586bfca5b3e989e] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp, totalSize=35.9 K 2024-11-28T06:56:06,478 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 4663aebb4c4f4d9db6f69e641bb1b486, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732776963222 2024-11-28T06:56:06,479 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting b2fe3d0082f443389b26ad75ab71d200, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732776964025 2024-11-28T06:56:06,479 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting a386b050049f4bce8586bfca5b3e989e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732776965281 2024-11-28T06:56:06,484 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00878c8327d194b967221c1933e24c9a#C#compaction#345 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:06,485 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/3ef4d7b93e6d480da667f8c4d6a2bfd9 is 50, key is test_row_0/C:col10/1732776965608/Put/seqid=0 2024-11-28T06:56:06,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742235_1411 (size=12561) 2024-11-28T06:56:06,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-28T06:56:06,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:06,535 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 00878c8327d194b967221c1933e24c9a 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-28T06:56:06,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=A 2024-11-28T06:56:06,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:56:06,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=B 2024-11-28T06:56:06,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:56:06,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=C 2024-11-28T06:56:06,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:56:06,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/25742a8b6e09410ea7194ea294e93d22 is 50, key is test_row_0/A:col10/1732776965763/Put/seqid=0 2024-11-28T06:56:06,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742236_1412 (size=12301) 2024-11-28T06:56:06,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-28T06:56:06,872 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/882bace7a22944cc94656ef870bc9876 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/882bace7a22944cc94656ef870bc9876 2024-11-28T06:56:06,875 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00878c8327d194b967221c1933e24c9a/A of 00878c8327d194b967221c1933e24c9a into 882bace7a22944cc94656ef870bc9876(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:06,875 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00878c8327d194b967221c1933e24c9a: 2024-11-28T06:56:06,875 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., storeName=00878c8327d194b967221c1933e24c9a/A, priority=13, startTime=1732776966452; duration=0sec 2024-11-28T06:56:06,876 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:06,876 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00878c8327d194b967221c1933e24c9a:A 2024-11-28T06:56:06,891 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/3ef4d7b93e6d480da667f8c4d6a2bfd9 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/3ef4d7b93e6d480da667f8c4d6a2bfd9 2024-11-28T06:56:06,894 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00878c8327d194b967221c1933e24c9a/C of 00878c8327d194b967221c1933e24c9a into 3ef4d7b93e6d480da667f8c4d6a2bfd9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:06,894 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00878c8327d194b967221c1933e24c9a: 2024-11-28T06:56:06,894 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., storeName=00878c8327d194b967221c1933e24c9a/C, priority=13, startTime=1732776966452; duration=0sec 2024-11-28T06:56:06,894 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:06,894 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00878c8327d194b967221c1933e24c9a:C 2024-11-28T06:56:06,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 00878c8327d194b967221c1933e24c9a 2024-11-28T06:56:06,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:06,945 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/25742a8b6e09410ea7194ea294e93d22 2024-11-28T06:56:06,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:06,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777026944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:06,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:06,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777026950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:06,956 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:06,956 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:06,956 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:06,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:06,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:06,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/e6c3cddd576242ceaea0e1aa6ec1340c is 50, key is test_row_0/B:col10/1732776965763/Put/seqid=0 2024-11-28T06:56:06,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:06,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742237_1413 (size=12301) 2024-11-28T06:56:07,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:07,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777027065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:07,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:07,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:07,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:07,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777027065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:07,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:07,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:07,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:07,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-28T06:56:07,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:07,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777027275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:07,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:07,282 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:07,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777027275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:07,282 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:07,282 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:07,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:07,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:07,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:07,364 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/e6c3cddd576242ceaea0e1aa6ec1340c 2024-11-28T06:56:07,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/e915a711d2864340b253915721e0dd89 is 50, key is test_row_0/C:col10/1732776965763/Put/seqid=0 2024-11-28T06:56:07,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742238_1414 (size=12301) 2024-11-28T06:56:07,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:07,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777027585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:07,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:07,591 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:07,591 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:07,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777027586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:07,591 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:07,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:07,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:07,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:07,775 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/e915a711d2864340b253915721e0dd89 2024-11-28T06:56:07,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/25742a8b6e09410ea7194ea294e93d22 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/25742a8b6e09410ea7194ea294e93d22 2024-11-28T06:56:07,781 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/25742a8b6e09410ea7194ea294e93d22, entries=150, sequenceid=294, filesize=12.0 K 2024-11-28T06:56:07,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/e6c3cddd576242ceaea0e1aa6ec1340c as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/e6c3cddd576242ceaea0e1aa6ec1340c 2024-11-28T06:56:07,785 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/e6c3cddd576242ceaea0e1aa6ec1340c, entries=150, sequenceid=294, filesize=12.0 K 2024-11-28T06:56:07,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.StoreScanner(997): StoreScanner already has the close lock. There is no need to updateReaders 2024-11-28T06:56:07,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/e915a711d2864340b253915721e0dd89 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/e915a711d2864340b253915721e0dd89 2024-11-28T06:56:07,790 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/e915a711d2864340b253915721e0dd89, entries=150, sequenceid=294, filesize=12.0 K 2024-11-28T06:56:07,790 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 00878c8327d194b967221c1933e24c9a in 1255ms, sequenceid=294, compaction requested=false 2024-11-28T06:56:07,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 00878c8327d194b967221c1933e24c9a: 2024-11-28T06:56:07,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:07,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-28T06:56:07,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-28T06:56:07,793 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-28T06:56:07,793 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7140 sec 2024-11-28T06:56:07,795 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.7180 sec 2024-11-28T06:56:08,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 00878c8327d194b967221c1933e24c9a 2024-11-28T06:56:08,101 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00878c8327d194b967221c1933e24c9a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T06:56:08,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=A 2024-11-28T06:56:08,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:08,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=B 2024-11-28T06:56:08,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:08,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=C 2024-11-28T06:56:08,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:08,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/9a7acb8f0d644a40907d4f736279c008 is 50, key is test_row_0/A:col10/1732776966937/Put/seqid=0 2024-11-28T06:56:08,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742239_1415 (size=14741) 2024-11-28T06:56:08,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732776968149, max timestamp=1732776968168Num uniques 859; 2024-11-28T06:56:08,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732776968149, max timestamp=1732776968168Num uniques 859; 2024-11-28T06:56:08,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T06:56:08,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T06:56:08,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732776968149, max timestamp=1732776968168Num uniques 859; 2024-11-28T06:56:08,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T06:56:08,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-28T06:56:08,226 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:56:08,226 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:56:08,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-28T06:56:08,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-28T06:56:08,228 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:56:08,228 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:56:08,228 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:56:08,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:08,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777028229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:08,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:08,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777028232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:08,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-28T06:56:08,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777028345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:08,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:08,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:08,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:08,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:08,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777028345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:08,354 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:08,354 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:08,354 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:08,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-28T06:56:08,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:08,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:08,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:08,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:08,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:08,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:08,508 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/9a7acb8f0d644a40907d4f736279c008 2024-11-28T06:56:08,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/981802e3ca5447389ebfd37025bc4113 is 50, key is test_row_0/B:col10/1732776966937/Put/seqid=0 2024-11-28T06:56:08,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742240_1416 (size=12301) 2024-11-28T06:56:08,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-28T06:56:08,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:08,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:08,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:08,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:08,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:08,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:08,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-28T06:56:08,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:08,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777028566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:08,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:08,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:08,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:08,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:08,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777028566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:08,578 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:08,578 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:08,578 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:08,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-28T06:56:08,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:08,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:08,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:08,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:08,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:08,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:08,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-28T06:56:08,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:08,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:08,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:08,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:08,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:08,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:08,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-28T06:56:08,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:08,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777028885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:08,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:08,896 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:08,896 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:08,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777028886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:08,896 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:08,896 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:08,896 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:08,896 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:08,917 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/981802e3ca5447389ebfd37025bc4113 2024-11-28T06:56:08,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/ea0d25c782ba490097cfbbd8a6dcda3e is 50, key is test_row_0/C:col10/1732776966937/Put/seqid=0 2024-11-28T06:56:08,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742241_1417 (size=12301) 2024-11-28T06:56:08,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-28T06:56:08,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:08,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:08,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:08,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:08,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:08,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:09,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-28T06:56:09,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:09,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:09,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:09,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:09,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:09,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:09,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-28T06:56:09,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:09,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:09,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:09,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:09,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:09,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/ea0d25c782ba490097cfbbd8a6dcda3e 2024-11-28T06:56:09,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/9a7acb8f0d644a40907d4f736279c008 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/9a7acb8f0d644a40907d4f736279c008 2024-11-28T06:56:09,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/9a7acb8f0d644a40907d4f736279c008, entries=200, sequenceid=307, filesize=14.4 K 2024-11-28T06:56:09,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/981802e3ca5447389ebfd37025bc4113 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/981802e3ca5447389ebfd37025bc4113 2024-11-28T06:56:09,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/981802e3ca5447389ebfd37025bc4113, entries=150, sequenceid=307, filesize=12.0 K 2024-11-28T06:56:09,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/ea0d25c782ba490097cfbbd8a6dcda3e as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/ea0d25c782ba490097cfbbd8a6dcda3e 2024-11-28T06:56:09,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/ea0d25c782ba490097cfbbd8a6dcda3e, entries=150, sequenceid=307, filesize=12.0 K 2024-11-28T06:56:09,343 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 00878c8327d194b967221c1933e24c9a in 1242ms, sequenceid=307, compaction requested=true 2024-11-28T06:56:09,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00878c8327d194b967221c1933e24c9a: 2024-11-28T06:56:09,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00878c8327d194b967221c1933e24c9a:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:56:09,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:09,343 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:56:09,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00878c8327d194b967221c1933e24c9a:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:56:09,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:09,343 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:56:09,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00878c8327d194b967221c1933e24c9a:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:56:09,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:56:09,344 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39603 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:56:09,344 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37163 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:56:09,344 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): 00878c8327d194b967221c1933e24c9a/A is initiating minor compaction (all files) 2024-11-28T06:56:09,344 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): 00878c8327d194b967221c1933e24c9a/B is initiating minor compaction (all files) 2024-11-28T06:56:09,344 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00878c8327d194b967221c1933e24c9a/A in TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:09,344 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00878c8327d194b967221c1933e24c9a/B in TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:09,344 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/882bace7a22944cc94656ef870bc9876, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/25742a8b6e09410ea7194ea294e93d22, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/9a7acb8f0d644a40907d4f736279c008] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp, totalSize=38.7 K 2024-11-28T06:56:09,344 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/ea0fc35b687e409294a2ca5d4c66fd06, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/e6c3cddd576242ceaea0e1aa6ec1340c, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/981802e3ca5447389ebfd37025bc4113] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp, totalSize=36.3 K 2024-11-28T06:56:09,344 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 882bace7a22944cc94656ef870bc9876, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732776965281 2024-11-28T06:56:09,344 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting ea0fc35b687e409294a2ca5d4c66fd06, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732776965281 2024-11-28T06:56:09,345 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25742a8b6e09410ea7194ea294e93d22, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732776965722 2024-11-28T06:56:09,345 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a7acb8f0d644a40907d4f736279c008, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732776966937 2024-11-28T06:56:09,345 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting e6c3cddd576242ceaea0e1aa6ec1340c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732776965722 2024-11-28T06:56:09,345 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 981802e3ca5447389ebfd37025bc4113, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732776966937 2024-11-28T06:56:09,350 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00878c8327d194b967221c1933e24c9a#A#compaction#352 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:09,350 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/8aaece6975ed413ca3aa11df6ad85fd3 is 50, key is test_row_0/A:col10/1732776966937/Put/seqid=0 2024-11-28T06:56:09,353 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00878c8327d194b967221c1933e24c9a#B#compaction#353 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:09,354 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/daffdc7060524e6492d10571c59bb448 is 50, key is test_row_0/B:col10/1732776966937/Put/seqid=0 2024-11-28T06:56:09,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742242_1418 (size=12813) 2024-11-28T06:56:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-28T06:56:09,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742243_1419 (size=12813) 2024-11-28T06:56:09,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 00878c8327d194b967221c1933e24c9a 2024-11-28T06:56:09,414 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00878c8327d194b967221c1933e24c9a 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-28T06:56:09,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=A 2024-11-28T06:56:09,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:56:09,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=B 2024-11-28T06:56:09,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:56:09,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=C 2024-11-28T06:56:09,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:56:09,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/b42eed0fb8b24c74bc978901fc5688cc is 50, key is test_row_0/A:col10/1732776968210/Put/seqid=0 2024-11-28T06:56:09,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742244_1420 (size=14741) 2024-11-28T06:56:09,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-28T06:56:09,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:09,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:09,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:09,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:09,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:09,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:09,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:09,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777029442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:09,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:09,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:09,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:09,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:09,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777029453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:09,456 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:09,456 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:09,456 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:09,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:09,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777029566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:09,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:09,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:09,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:09,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:09,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777029566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:09,569 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:09,569 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:09,569 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:09,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-28T06:56:09,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:09,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:09,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:09,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:09,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:09,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:09,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-28T06:56:09,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:09,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:09,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:09,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:09,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:09,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:09,763 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/8aaece6975ed413ca3aa11df6ad85fd3 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/8aaece6975ed413ca3aa11df6ad85fd3 2024-11-28T06:56:09,767 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00878c8327d194b967221c1933e24c9a/A of 00878c8327d194b967221c1933e24c9a into 8aaece6975ed413ca3aa11df6ad85fd3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:09,767 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00878c8327d194b967221c1933e24c9a: 2024-11-28T06:56:09,767 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., storeName=00878c8327d194b967221c1933e24c9a/A, priority=13, startTime=1732776969343; duration=0sec 2024-11-28T06:56:09,767 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:56:09,767 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00878c8327d194b967221c1933e24c9a:A 2024-11-28T06:56:09,767 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:56:09,767 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37163 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:56:09,767 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): 00878c8327d194b967221c1933e24c9a/C is initiating minor compaction (all files) 2024-11-28T06:56:09,767 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00878c8327d194b967221c1933e24c9a/C in TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:09,768 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/3ef4d7b93e6d480da667f8c4d6a2bfd9, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/e915a711d2864340b253915721e0dd89, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/ea0d25c782ba490097cfbbd8a6dcda3e] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp, totalSize=36.3 K 2024-11-28T06:56:09,768 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ef4d7b93e6d480da667f8c4d6a2bfd9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732776965281 2024-11-28T06:56:09,768 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting e915a711d2864340b253915721e0dd89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732776965722 2024-11-28T06:56:09,768 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea0d25c782ba490097cfbbd8a6dcda3e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732776966937 2024-11-28T06:56:09,769 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/daffdc7060524e6492d10571c59bb448 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/daffdc7060524e6492d10571c59bb448 2024-11-28T06:56:09,772 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00878c8327d194b967221c1933e24c9a/B of 00878c8327d194b967221c1933e24c9a into daffdc7060524e6492d10571c59bb448(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:09,773 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00878c8327d194b967221c1933e24c9a: 2024-11-28T06:56:09,773 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., storeName=00878c8327d194b967221c1933e24c9a/B, priority=13, startTime=1732776969343; duration=0sec 2024-11-28T06:56:09,773 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:09,773 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00878c8327d194b967221c1933e24c9a:B 2024-11-28T06:56:09,773 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00878c8327d194b967221c1933e24c9a#C#compaction#355 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:09,774 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/c53401925fbf431fb7fb66648b940c40 is 50, key is test_row_0/C:col10/1732776966937/Put/seqid=0 2024-11-28T06:56:09,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742245_1421 (size=12813) 2024-11-28T06:56:09,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:09,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777029775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:09,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:09,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:09,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:09,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:09,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777029776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:09,791 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:09,791 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:09,791 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:09,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/b42eed0fb8b24c74bc978901fc5688cc 2024-11-28T06:56:09,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/62f7acb609bf4bc483ffb76b5f2dce80 is 50, key is test_row_0/B:col10/1732776968210/Put/seqid=0 2024-11-28T06:56:09,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742246_1422 (size=12301) 2024-11-28T06:56:09,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-28T06:56:09,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:09,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:09,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:09,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:09,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:09,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:10,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-28T06:56:10,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:10,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:10,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:10,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:10,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:10,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:10,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:10,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777030095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:10,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:10,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:10,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777030095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:10,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:10,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:10,101 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:10,101 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:10,101 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:10,181 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/c53401925fbf431fb7fb66648b940c40 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/c53401925fbf431fb7fb66648b940c40 2024-11-28T06:56:10,184 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00878c8327d194b967221c1933e24c9a/C of 00878c8327d194b967221c1933e24c9a into c53401925fbf431fb7fb66648b940c40(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:10,184 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00878c8327d194b967221c1933e24c9a: 2024-11-28T06:56:10,184 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., storeName=00878c8327d194b967221c1933e24c9a/C, priority=13, startTime=1732776969343; duration=0sec 2024-11-28T06:56:10,184 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:10,184 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00878c8327d194b967221c1933e24c9a:C 2024-11-28T06:56:10,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-28T06:56:10,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:10,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:10,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:10,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:10,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:10,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:10,232 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/62f7acb609bf4bc483ffb76b5f2dce80 2024-11-28T06:56:10,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/e624ba8f0e0a42219d0711d02570cd2b is 50, key is test_row_0/C:col10/1732776968210/Put/seqid=0 2024-11-28T06:56:10,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742247_1423 (size=12301) 2024-11-28T06:56:10,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-28T06:56:10,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:10,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:10,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:10,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:10,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:10,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:10,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-28T06:56:10,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-28T06:56:10,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:10,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:10,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:10,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:10,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:10,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:10,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:10,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777030606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:10,613 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:10,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:10,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:10,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:10,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777030606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:10,614 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:10,614 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:10,614 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:10,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/e624ba8f0e0a42219d0711d02570cd2b 2024-11-28T06:56:10,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/b42eed0fb8b24c74bc978901fc5688cc as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/b42eed0fb8b24c74bc978901fc5688cc 2024-11-28T06:56:10,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/b42eed0fb8b24c74bc978901fc5688cc, entries=200, sequenceid=343, filesize=14.4 K 2024-11-28T06:56:10,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/62f7acb609bf4bc483ffb76b5f2dce80 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/62f7acb609bf4bc483ffb76b5f2dce80 2024-11-28T06:56:10,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/62f7acb609bf4bc483ffb76b5f2dce80, entries=150, sequenceid=343, filesize=12.0 K 2024-11-28T06:56:10,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/e624ba8f0e0a42219d0711d02570cd2b as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/e624ba8f0e0a42219d0711d02570cd2b 2024-11-28T06:56:10,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/e624ba8f0e0a42219d0711d02570cd2b, entries=150, sequenceid=343, filesize=12.0 K 2024-11-28T06:56:10,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 00878c8327d194b967221c1933e24c9a in 1242ms, sequenceid=343, compaction requested=false 2024-11-28T06:56:10,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00878c8327d194b967221c1933e24c9a: 2024-11-28T06:56:10,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-28T06:56:10,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:10,670 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 00878c8327d194b967221c1933e24c9a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T06:56:10,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=A 2024-11-28T06:56:10,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:10,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=B 2024-11-28T06:56:10,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:10,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=C 2024-11-28T06:56:10,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:10,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/d9cbdf578b474b8196e718f3c593c47e is 50, key is test_row_0/A:col10/1732776969441/Put/seqid=0 2024-11-28T06:56:10,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742248_1424 (size=12301) 2024-11-28T06:56:11,077 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/d9cbdf578b474b8196e718f3c593c47e 2024-11-28T06:56:11,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/1538da6afe66443ca0291589b97df0ca is 50, key is test_row_0/B:col10/1732776969441/Put/seqid=0 2024-11-28T06:56:11,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742249_1425 (size=12301) 2024-11-28T06:56:11,486 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/1538da6afe66443ca0291589b97df0ca 2024-11-28T06:56:11,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/d8b33af03b114947abc4acb47957614f is 50, key is test_row_0/C:col10/1732776969441/Put/seqid=0 2024-11-28T06:56:11,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742250_1426 (size=12301) 2024-11-28T06:56:11,536 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/d8b33af03b114947abc4acb47957614f 2024-11-28T06:56:11,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/d9cbdf578b474b8196e718f3c593c47e as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/d9cbdf578b474b8196e718f3c593c47e 2024-11-28T06:56:11,545 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/d9cbdf578b474b8196e718f3c593c47e, entries=150, sequenceid=356, filesize=12.0 K 2024-11-28T06:56:11,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/1538da6afe66443ca0291589b97df0ca as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/1538da6afe66443ca0291589b97df0ca 2024-11-28T06:56:11,549 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/1538da6afe66443ca0291589b97df0ca, entries=150, sequenceid=356, filesize=12.0 K 2024-11-28T06:56:11,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/d8b33af03b114947abc4acb47957614f as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/d8b33af03b114947abc4acb47957614f 2024-11-28T06:56:11,553 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/d8b33af03b114947abc4acb47957614f, entries=150, sequenceid=356, filesize=12.0 K 2024-11-28T06:56:11,553 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for 00878c8327d194b967221c1933e24c9a in 883ms, sequenceid=356, compaction requested=true 2024-11-28T06:56:11,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 00878c8327d194b967221c1933e24c9a: 2024-11-28T06:56:11,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:11,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-28T06:56:11,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-28T06:56:11,558 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-28T06:56:11,558 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3280 sec 2024-11-28T06:56:11,560 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 3.3320 sec 2024-11-28T06:56:11,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 00878c8327d194b967221c1933e24c9a 2024-11-28T06:56:11,639 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00878c8327d194b967221c1933e24c9a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T06:56:11,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=A 2024-11-28T06:56:11,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:11,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=B 2024-11-28T06:56:11,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:11,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=C 2024-11-28T06:56:11,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:11,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/c6a52b677f7a4ba3850c08caa14477c3 is 50, key is test_row_0/A:col10/1732776971636/Put/seqid=0 2024-11-28T06:56:11,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742251_1427 (size=19621) 2024-11-28T06:56:11,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732776971736, max timestamp=1732776971771Num uniques 859; 2024-11-28T06:56:11,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732776971736, max timestamp=1732776971771Num uniques 859; 2024-11-28T06:56:11,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732776971736, max timestamp=1732776971771Num uniques 859; 2024-11-28T06:56:11,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T06:56:11,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T06:56:11,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T06:56:11,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:11,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777031855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:11,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:11,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:11,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:11,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:11,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777031857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:11,870 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:11,870 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:11,870 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:11,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:11,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777031975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:11,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:11,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:11,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:11,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777031975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:11,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:11,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:11,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:11,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:12,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/c6a52b677f7a4ba3850c08caa14477c3 2024-11-28T06:56:12,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/2d1968ec82414f75860fb4c408d05853 is 50, key is test_row_0/B:col10/1732776971636/Put/seqid=0 2024-11-28T06:56:12,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742252_1428 (size=12301) 2024-11-28T06:56:12,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:12,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777032195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:12,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:12,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:12,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777032195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:12,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:12,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:12,202 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:12,202 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:12,202 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:12,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-28T06:56:12,386 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:56:12,386 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:56:12,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-28T06:56:12,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-28T06:56:12,388 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:56:12,389 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:56:12,389 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:56:12,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/2d1968ec82414f75860fb4c408d05853 2024-11-28T06:56:12,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/b33cb4d59bfc41e18ea2dae185ada085 is 50, key is test_row_0/C:col10/1732776971636/Put/seqid=0 2024-11-28T06:56:12,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742253_1429 (size=12301) 2024-11-28T06:56:12,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-28T06:56:12,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:12,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777032505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:12,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:12,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777032505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:12,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:12,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:12,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:12,517 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:12,517 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:12,517 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:12,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-28T06:56:12,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:12,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:12,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:12,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:12,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:12,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:12,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-28T06:56:12,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:12,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:12,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:12,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:12,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:12,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:12,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-28T06:56:12,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-28T06:56:12,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:12,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:12,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:12,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:12,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:12,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:12,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/b33cb4d59bfc41e18ea2dae185ada085 2024-11-28T06:56:12,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/c6a52b677f7a4ba3850c08caa14477c3 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/c6a52b677f7a4ba3850c08caa14477c3 2024-11-28T06:56:12,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/c6a52b677f7a4ba3850c08caa14477c3, entries=300, sequenceid=366, filesize=19.2 K 2024-11-28T06:56:12,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/2d1968ec82414f75860fb4c408d05853 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/2d1968ec82414f75860fb4c408d05853 2024-11-28T06:56:12,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/2d1968ec82414f75860fb4c408d05853, entries=150, sequenceid=366, filesize=12.0 K 2024-11-28T06:56:12,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/b33cb4d59bfc41e18ea2dae185ada085 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/b33cb4d59bfc41e18ea2dae185ada085 2024-11-28T06:56:12,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/b33cb4d59bfc41e18ea2dae185ada085, entries=150, sequenceid=366, filesize=12.0 K 2024-11-28T06:56:12,880 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 00878c8327d194b967221c1933e24c9a in 1241ms, sequenceid=366, compaction requested=true 2024-11-28T06:56:12,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00878c8327d194b967221c1933e24c9a: 2024-11-28T06:56:12,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00878c8327d194b967221c1933e24c9a:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:56:12,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:12,881 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T06:56:12,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00878c8327d194b967221c1933e24c9a:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:56:12,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:12,881 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T06:56:12,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00878c8327d194b967221c1933e24c9a:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:56:12,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:56:12,881 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49716 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T06:56:12,881 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 59476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T06:56:12,881 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): 00878c8327d194b967221c1933e24c9a/A is initiating minor compaction (all files) 2024-11-28T06:56:12,881 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): 00878c8327d194b967221c1933e24c9a/B is initiating minor compaction (all files) 2024-11-28T06:56:12,881 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00878c8327d194b967221c1933e24c9a/B in TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:12,882 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00878c8327d194b967221c1933e24c9a/A in TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:12,882 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/daffdc7060524e6492d10571c59bb448, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/62f7acb609bf4bc483ffb76b5f2dce80, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/1538da6afe66443ca0291589b97df0ca, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/2d1968ec82414f75860fb4c408d05853] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp, totalSize=48.6 K 2024-11-28T06:56:12,882 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/8aaece6975ed413ca3aa11df6ad85fd3, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/b42eed0fb8b24c74bc978901fc5688cc, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/d9cbdf578b474b8196e718f3c593c47e, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/c6a52b677f7a4ba3850c08caa14477c3] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp, totalSize=58.1 K 2024-11-28T06:56:12,882 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8aaece6975ed413ca3aa11df6ad85fd3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732776966937 2024-11-28T06:56:12,882 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting daffdc7060524e6492d10571c59bb448, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732776966937 2024-11-28T06:56:12,882 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting b42eed0fb8b24c74bc978901fc5688cc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732776968210 2024-11-28T06:56:12,882 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 62f7acb609bf4bc483ffb76b5f2dce80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732776968210 2024-11-28T06:56:12,882 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9cbdf578b474b8196e718f3c593c47e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732776969420 2024-11-28T06:56:12,882 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 1538da6afe66443ca0291589b97df0ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732776969420 2024-11-28T06:56:12,882 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6a52b677f7a4ba3850c08caa14477c3, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1732776971627 2024-11-28T06:56:12,882 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d1968ec82414f75860fb4c408d05853, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1732776971628 2024-11-28T06:56:12,891 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00878c8327d194b967221c1933e24c9a#B#compaction#364 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:12,891 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00878c8327d194b967221c1933e24c9a#A#compaction#365 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:12,892 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/f79b58d3df104bd49778a79c2934d021 is 50, key is test_row_0/B:col10/1732776971636/Put/seqid=0 2024-11-28T06:56:12,892 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/98d87534165e485cafe8228d4638f939 is 50, key is test_row_0/A:col10/1732776971636/Put/seqid=0 2024-11-28T06:56:12,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742255_1431 (size=12949) 2024-11-28T06:56:12,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742254_1430 (size=12949) 2024-11-28T06:56:12,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-28T06:56:12,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:12,999 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 00878c8327d194b967221c1933e24c9a 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-28T06:56:12,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=A 2024-11-28T06:56:13,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:56:13,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=B 2024-11-28T06:56:13,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:56:13,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=C 2024-11-28T06:56:13,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:56:13,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/7b3787322e11492d848da2f784cbb9f4 is 50, key is test_row_0/A:col10/1732776971854/Put/seqid=0 2024-11-28T06:56:13,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742256_1432 (size=12301) 2024-11-28T06:56:13,008 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/7b3787322e11492d848da2f784cbb9f4 2024-11-28T06:56:13,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/bc06cf89605d4c259a39502927f04024 is 50, key is test_row_0/B:col10/1732776971854/Put/seqid=0 2024-11-28T06:56:13,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-28T06:56:13,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742257_1433 (size=12301) 2024-11-28T06:56:13,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 00878c8327d194b967221c1933e24c9a 2024-11-28T06:56:13,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:13,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:13,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777033065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:13,074 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:13,074 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:13,074 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:13,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:13,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777033072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:13,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:13,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:13,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:13,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:13,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777033186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:13,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:13,196 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:13,196 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:13,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777033186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:13,196 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:13,196 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:13,196 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:13,196 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:13,299 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/98d87534165e485cafe8228d4638f939 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/98d87534165e485cafe8228d4638f939 2024-11-28T06:56:13,302 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/f79b58d3df104bd49778a79c2934d021 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/f79b58d3df104bd49778a79c2934d021 2024-11-28T06:56:13,303 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 00878c8327d194b967221c1933e24c9a/A of 00878c8327d194b967221c1933e24c9a into 98d87534165e485cafe8228d4638f939(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:13,303 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00878c8327d194b967221c1933e24c9a: 2024-11-28T06:56:13,303 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., storeName=00878c8327d194b967221c1933e24c9a/A, priority=12, startTime=1732776972881; duration=0sec 2024-11-28T06:56:13,303 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:56:13,303 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00878c8327d194b967221c1933e24c9a:A 2024-11-28T06:56:13,304 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T06:56:13,304 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49716 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T06:56:13,305 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): 00878c8327d194b967221c1933e24c9a/C is initiating minor compaction (all files) 2024-11-28T06:56:13,305 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00878c8327d194b967221c1933e24c9a/C in TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:13,305 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/c53401925fbf431fb7fb66648b940c40, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/e624ba8f0e0a42219d0711d02570cd2b, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/d8b33af03b114947abc4acb47957614f, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/b33cb4d59bfc41e18ea2dae185ada085] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp, totalSize=48.6 K 2024-11-28T06:56:13,305 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting c53401925fbf431fb7fb66648b940c40, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732776966937 2024-11-28T06:56:13,305 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting e624ba8f0e0a42219d0711d02570cd2b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732776968210 2024-11-28T06:56:13,306 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8b33af03b114947abc4acb47957614f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732776969420 2024-11-28T06:56:13,306 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting b33cb4d59bfc41e18ea2dae185ada085, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1732776971628 2024-11-28T06:56:13,309 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 00878c8327d194b967221c1933e24c9a/B of 00878c8327d194b967221c1933e24c9a into f79b58d3df104bd49778a79c2934d021(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:13,309 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00878c8327d194b967221c1933e24c9a: 2024-11-28T06:56:13,309 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., storeName=00878c8327d194b967221c1933e24c9a/B, priority=12, startTime=1732776972881; duration=0sec 2024-11-28T06:56:13,309 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:13,309 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00878c8327d194b967221c1933e24c9a:B 2024-11-28T06:56:13,315 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00878c8327d194b967221c1933e24c9a#C#compaction#368 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:13,315 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/c1daa23b3f234f72a3718001501a487e is 50, key is test_row_0/C:col10/1732776971636/Put/seqid=0 2024-11-28T06:56:13,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742258_1434 (size=12949) 2024-11-28T06:56:13,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:13,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777033405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:13,412 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:13,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:13,412 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:13,412 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:13,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777033405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:13,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:13,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:13,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:13,418 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/bc06cf89605d4c259a39502927f04024 2024-11-28T06:56:13,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/50ac20973553435f92c77c89e576a13f is 50, key is test_row_0/C:col10/1732776971854/Put/seqid=0 2024-11-28T06:56:13,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742259_1435 (size=12301) 2024-11-28T06:56:13,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-28T06:56:13,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:13,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777033715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:13,720 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:13,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:13,720 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:13,720 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:13,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777033715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:13,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:13,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:13,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:13,722 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/c1daa23b3f234f72a3718001501a487e as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/c1daa23b3f234f72a3718001501a487e 2024-11-28T06:56:13,726 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 00878c8327d194b967221c1933e24c9a/C of 00878c8327d194b967221c1933e24c9a into c1daa23b3f234f72a3718001501a487e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:13,726 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00878c8327d194b967221c1933e24c9a: 2024-11-28T06:56:13,726 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., storeName=00878c8327d194b967221c1933e24c9a/C, priority=12, startTime=1732776972881; duration=0sec 2024-11-28T06:56:13,726 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:13,726 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00878c8327d194b967221c1933e24c9a:C 2024-11-28T06:56:13,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:13,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46398 deadline: 1732777033735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:13,745 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_0' in 00878c8327d194b967221c1933e24c9a of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18335 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:56:13,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:13,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:13,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:13,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:13,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46364 deadline: 1732777033766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:13,772 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_0' in 00878c8327d194b967221c1933e24c9a of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18306 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:56:13,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:13,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:13,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:13,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:13,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46420 deadline: 1732777033775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:13,788 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_1' in 00878c8327d194b967221c1933e24c9a of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18375 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:56:13,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:13,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:13,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:13,827 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/50ac20973553435f92c77c89e576a13f 2024-11-28T06:56:13,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/7b3787322e11492d848da2f784cbb9f4 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/7b3787322e11492d848da2f784cbb9f4 2024-11-28T06:56:13,834 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/7b3787322e11492d848da2f784cbb9f4, entries=150, sequenceid=402, filesize=12.0 K 2024-11-28T06:56:13,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/bc06cf89605d4c259a39502927f04024 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/bc06cf89605d4c259a39502927f04024 2024-11-28T06:56:13,837 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/bc06cf89605d4c259a39502927f04024, entries=150, sequenceid=402, filesize=12.0 K 2024-11-28T06:56:13,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/50ac20973553435f92c77c89e576a13f as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/50ac20973553435f92c77c89e576a13f 2024-11-28T06:56:13,841 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/50ac20973553435f92c77c89e576a13f, entries=150, sequenceid=402, filesize=12.0 K 2024-11-28T06:56:13,841 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 00878c8327d194b967221c1933e24c9a in 842ms, sequenceid=402, compaction requested=false 2024-11-28T06:56:13,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 00878c8327d194b967221c1933e24c9a: 2024-11-28T06:56:13,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:13,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-28T06:56:13,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-28T06:56:13,844 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-28T06:56:13,844 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4530 sec 2024-11-28T06:56:13,845 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.4580 sec 2024-11-28T06:56:13,869 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T06:56:14,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 00878c8327d194b967221c1933e24c9a 2024-11-28T06:56:14,230 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00878c8327d194b967221c1933e24c9a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T06:56:14,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=A 2024-11-28T06:56:14,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:14,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=B 2024-11-28T06:56:14,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:14,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=C 2024-11-28T06:56:14,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:14,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/253eaa0650da4414a9cdeb865e287f37 is 50, key is test_row_0/A:col10/1732776974229/Put/seqid=0 2024-11-28T06:56:14,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742260_1436 (size=14741) 2024-11-28T06:56:14,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:14,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777034299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:14,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732776974269, max timestamp=1732776974305Num uniques 859; 2024-11-28T06:56:14,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T06:56:14,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732776974269, max timestamp=1732776974305Num uniques 859; 2024-11-28T06:56:14,308 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:14,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732776974269, max timestamp=1732776974305Num uniques 859; 2024-11-28T06:56:14,308 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:14,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T06:56:14,308 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:14,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T06:56:14,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:14,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777034425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:14,434 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:14,434 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:14,434 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:14,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:14,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777034430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:14,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:14,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:14,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:14,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-28T06:56:14,536 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:56:14,536 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:56:14,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-28T06:56:14,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-28T06:56:14,538 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:56:14,539 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:56:14,539 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:56:14,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:14,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777034536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:14,544 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:14,544 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:14,544 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:14,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:14,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777034545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:14,548 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:14,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:14,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:14,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/253eaa0650da4414a9cdeb865e287f37 2024-11-28T06:56:14,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-28T06:56:14,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/c6b1b3325d4140659f202eac1463490c is 50, key is test_row_0/B:col10/1732776974229/Put/seqid=0 2024-11-28T06:56:14,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742261_1437 (size=12301) 2024-11-28T06:56:14,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-28T06:56:14,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:14,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:14,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:14,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:14,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:14,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:14,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777034755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:14,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:14,764 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:14,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777034755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:14,764 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:14,764 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:14,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:14,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:14,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:14,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-28T06:56:14,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:14,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:14,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:14,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:14,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:14,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:14,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-28T06:56:14,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-28T06:56:14,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:14,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:14,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:14,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:14,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:14,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:15,050 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/c6b1b3325d4140659f202eac1463490c 2024-11-28T06:56:15,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/ec2657904ddc433187db560ce1a46609 is 50, key is test_row_0/C:col10/1732776974229/Put/seqid=0 2024-11-28T06:56:15,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742262_1438 (size=12301) 2024-11-28T06:56:15,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:15,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46450 deadline: 1732777035075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:15,087 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:15,087 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:15,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:15,087 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:15,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46426 deadline: 1732777035076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:15,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 , the old value is region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:15,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:15,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., hostname=1cb9cf437a36,43341,1732776825568, seqNum=2 because the exception is null or not the one we care about 2024-11-28T06:56:15,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-28T06:56:15,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:15,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:15,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:15,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:15,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:15,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:15,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-28T06:56:15,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-28T06:56:15,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:15,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:15,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:15,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:15,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:15,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:15,337 INFO [Thread-1720 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1720. 2024-11-28T06:56:15,338 DEBUG [Thread-1720 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:56:15,338 DEBUG [Thread-1720 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:15,338 DEBUG [Thread-1720 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:15,338 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:56:15,339 INFO [Thread-1722 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1722. 2024-11-28T06:56:15,339 DEBUG [Thread-1722 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:56:15,339 DEBUG [Thread-1722 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:15,339 DEBUG [Thread-1722 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:15,339 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:56:15,343 INFO [Thread-1728 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1728. 2024-11-28T06:56:15,343 DEBUG [Thread-1728 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:56:15,343 DEBUG [Thread-1728 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:15,344 DEBUG [Thread-1728 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:15,344 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:56:15,350 INFO [Thread-1726 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1726. 2024-11-28T06:56:15,350 DEBUG [Thread-1726 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:56:15,350 DEBUG [Thread-1726 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:15,350 DEBUG [Thread-1726 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:15,350 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:56:15,353 INFO [Thread-1724 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1724. 2024-11-28T06:56:15,353 DEBUG [Thread-1724 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:56:15,353 DEBUG [Thread-1724 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:15,353 DEBUG [Thread-1724 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:15,353 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:56:15,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-28T06:56:15,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:15,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:15,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:15,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:15,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:15,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:15,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/ec2657904ddc433187db560ce1a46609 2024-11-28T06:56:15,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/253eaa0650da4414a9cdeb865e287f37 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/253eaa0650da4414a9cdeb865e287f37 2024-11-28T06:56:15,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/253eaa0650da4414a9cdeb865e287f37, entries=200, sequenceid=416, filesize=14.4 K 2024-11-28T06:56:15,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/c6b1b3325d4140659f202eac1463490c as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/c6b1b3325d4140659f202eac1463490c 2024-11-28T06:56:15,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/c6b1b3325d4140659f202eac1463490c, entries=150, sequenceid=416, filesize=12.0 K 2024-11-28T06:56:15,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/ec2657904ddc433187db560ce1a46609 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/ec2657904ddc433187db560ce1a46609 2024-11-28T06:56:15,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/ec2657904ddc433187db560ce1a46609, entries=150, sequenceid=416, filesize=12.0 K 2024-11-28T06:56:15,478 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 00878c8327d194b967221c1933e24c9a in 1248ms, sequenceid=416, compaction requested=true 2024-11-28T06:56:15,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00878c8327d194b967221c1933e24c9a: 2024-11-28T06:56:15,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00878c8327d194b967221c1933e24c9a:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:56:15,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:15,479 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:56:15,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00878c8327d194b967221c1933e24c9a:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:56:15,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:15,479 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:56:15,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00878c8327d194b967221c1933e24c9a:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:56:15,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:56:15,480 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:56:15,480 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): 00878c8327d194b967221c1933e24c9a/A is initiating minor compaction (all files) 2024-11-28T06:56:15,480 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:56:15,480 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00878c8327d194b967221c1933e24c9a/A in TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:15,480 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): 00878c8327d194b967221c1933e24c9a/B is initiating minor compaction (all files) 2024-11-28T06:56:15,480 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00878c8327d194b967221c1933e24c9a/B in TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:15,480 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/98d87534165e485cafe8228d4638f939, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/7b3787322e11492d848da2f784cbb9f4, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/253eaa0650da4414a9cdeb865e287f37] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp, totalSize=39.1 K 2024-11-28T06:56:15,480 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/f79b58d3df104bd49778a79c2934d021, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/bc06cf89605d4c259a39502927f04024, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/c6b1b3325d4140659f202eac1463490c] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp, totalSize=36.7 K 2024-11-28T06:56:15,480 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98d87534165e485cafe8228d4638f939, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1732776971628 2024-11-28T06:56:15,481 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting f79b58d3df104bd49778a79c2934d021, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1732776971628 2024-11-28T06:56:15,481 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b3787322e11492d848da2f784cbb9f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732776971840 2024-11-28T06:56:15,481 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting bc06cf89605d4c259a39502927f04024, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732776971840 2024-11-28T06:56:15,481 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 253eaa0650da4414a9cdeb865e287f37, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732776973054 2024-11-28T06:56:15,481 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting c6b1b3325d4140659f202eac1463490c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732776973054 2024-11-28T06:56:15,488 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00878c8327d194b967221c1933e24c9a#B#compaction#373 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:15,488 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00878c8327d194b967221c1933e24c9a#A#compaction#374 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:15,489 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/ca9c36bf8b2d403bb30089150d1263b2 is 50, key is test_row_0/B:col10/1732776974229/Put/seqid=0 2024-11-28T06:56:15,489 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/563ee429a0e74483bc334c0dc21d0f76 is 50, key is test_row_0/A:col10/1732776974229/Put/seqid=0 2024-11-28T06:56:15,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742264_1440 (size=13051) 2024-11-28T06:56:15,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742263_1439 (size=13051) 2024-11-28T06:56:15,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on 00878c8327d194b967221c1933e24c9a 2024-11-28T06:56:15,601 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00878c8327d194b967221c1933e24c9a 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-28T06:56:15,601 INFO [Thread-1713 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1713. 2024-11-28T06:56:15,601 INFO [Thread-1717 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1717. 2024-11-28T06:56:15,601 DEBUG [Thread-1713 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:56:15,601 DEBUG [Thread-1717 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:56:15,601 DEBUG [Thread-1713 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:15,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=A 2024-11-28T06:56:15,601 DEBUG [Thread-1717 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:15,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:56:15,601 DEBUG [Thread-1713 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:15,601 DEBUG [Thread-1717 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:15,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=B 2024-11-28T06:56:15,602 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:56:15,602 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:56:15,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:56:15,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=C 2024-11-28T06:56:15,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:56:15,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-28T06:56:15,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:15,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:15,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:15,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:15,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:15,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:15,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/14ccd5ab960d45e9aade3f8dd74c57b3 is 50, key is test_row_0/A:col10/1732776975597/Put/seqid=0 2024-11-28T06:56:15,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742265_1441 (size=12301) 2024-11-28T06:56:15,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-28T06:56:15,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-28T06:56:15,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:15,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:15,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:15,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:15,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:15,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:15,901 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/ca9c36bf8b2d403bb30089150d1263b2 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/ca9c36bf8b2d403bb30089150d1263b2 2024-11-28T06:56:15,901 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/563ee429a0e74483bc334c0dc21d0f76 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/563ee429a0e74483bc334c0dc21d0f76 2024-11-28T06:56:15,906 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00878c8327d194b967221c1933e24c9a/B of 00878c8327d194b967221c1933e24c9a into ca9c36bf8b2d403bb30089150d1263b2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:15,906 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00878c8327d194b967221c1933e24c9a/A of 00878c8327d194b967221c1933e24c9a into 563ee429a0e74483bc334c0dc21d0f76(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:15,906 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00878c8327d194b967221c1933e24c9a: 2024-11-28T06:56:15,906 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00878c8327d194b967221c1933e24c9a: 2024-11-28T06:56:15,906 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., storeName=00878c8327d194b967221c1933e24c9a/A, priority=13, startTime=1732776975479; duration=0sec 2024-11-28T06:56:15,906 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., storeName=00878c8327d194b967221c1933e24c9a/B, priority=13, startTime=1732776975479; duration=0sec 2024-11-28T06:56:15,907 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:56:15,907 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00878c8327d194b967221c1933e24c9a:B 2024-11-28T06:56:15,907 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:15,907 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00878c8327d194b967221c1933e24c9a:A 2024-11-28T06:56:15,907 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:56:15,907 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:56:15,908 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): 00878c8327d194b967221c1933e24c9a/C is initiating minor compaction (all files) 2024-11-28T06:56:15,908 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00878c8327d194b967221c1933e24c9a/C in TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:15,908 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/c1daa23b3f234f72a3718001501a487e, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/50ac20973553435f92c77c89e576a13f, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/ec2657904ddc433187db560ce1a46609] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp, totalSize=36.7 K 2024-11-28T06:56:15,908 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting c1daa23b3f234f72a3718001501a487e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1732776971628 2024-11-28T06:56:15,908 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 50ac20973553435f92c77c89e576a13f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732776971840 2024-11-28T06:56:15,909 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting ec2657904ddc433187db560ce1a46609, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732776973054 2024-11-28T06:56:15,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-28T06:56:15,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:15,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:15,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:15,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:15,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:15,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:15,917 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00878c8327d194b967221c1933e24c9a#C#compaction#376 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:15,918 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/f0bda250fa98442085c90311a84dd87a is 50, key is test_row_0/C:col10/1732776974229/Put/seqid=0 2024-11-28T06:56:15,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742266_1442 (size=13051) 2024-11-28T06:56:16,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/14ccd5ab960d45e9aade3f8dd74c57b3 2024-11-28T06:56:16,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/59100959a3a4409fabbf5d095a2f1640 is 50, key is test_row_0/B:col10/1732776975597/Put/seqid=0 2024-11-28T06:56:16,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742267_1443 (size=12301) 2024-11-28T06:56:16,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-28T06:56:16,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:16,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:16,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:16,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:16,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:16,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:16,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-28T06:56:16,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:16,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:16,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:16,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:16,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:16,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:16,328 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/f0bda250fa98442085c90311a84dd87a as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/f0bda250fa98442085c90311a84dd87a 2024-11-28T06:56:16,334 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00878c8327d194b967221c1933e24c9a/C of 00878c8327d194b967221c1933e24c9a into f0bda250fa98442085c90311a84dd87a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:16,334 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00878c8327d194b967221c1933e24c9a: 2024-11-28T06:56:16,334 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., storeName=00878c8327d194b967221c1933e24c9a/C, priority=13, startTime=1732776975479; duration=0sec 2024-11-28T06:56:16,334 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:16,334 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00878c8327d194b967221c1933e24c9a:C 2024-11-28T06:56:16,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-28T06:56:16,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:16,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:16,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:16,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:16,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:16,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:16,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/59100959a3a4409fabbf5d095a2f1640 2024-11-28T06:56:16,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/fa8eb46c308648ed93f91c6f73bb9113 is 50, key is test_row_0/C:col10/1732776975597/Put/seqid=0 2024-11-28T06:56:16,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742268_1444 (size=12301) 2024-11-28T06:56:16,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-28T06:56:16,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:16,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:16,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:16,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:16,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:16,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:16,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-28T06:56:16,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:16,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:16,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:16,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:16,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:16,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:16,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-28T06:56:16,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-28T06:56:16,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:16,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. as already flushing 2024-11-28T06:56:16,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:16,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:16,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:16,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:16,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/fa8eb46c308648ed93f91c6f73bb9113 2024-11-28T06:56:16,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/14ccd5ab960d45e9aade3f8dd74c57b3 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/14ccd5ab960d45e9aade3f8dd74c57b3 2024-11-28T06:56:16,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/14ccd5ab960d45e9aade3f8dd74c57b3, entries=150, sequenceid=453, filesize=12.0 K 2024-11-28T06:56:16,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/59100959a3a4409fabbf5d095a2f1640 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/59100959a3a4409fabbf5d095a2f1640 2024-11-28T06:56:16,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/59100959a3a4409fabbf5d095a2f1640, entries=150, sequenceid=453, filesize=12.0 K 2024-11-28T06:56:16,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/fa8eb46c308648ed93f91c6f73bb9113 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/fa8eb46c308648ed93f91c6f73bb9113 2024-11-28T06:56:16,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/fa8eb46c308648ed93f91c6f73bb9113, entries=150, sequenceid=453, filesize=12.0 K 2024-11-28T06:56:16,854 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=0 B/0 for 00878c8327d194b967221c1933e24c9a in 1253ms, sequenceid=453, compaction requested=false 2024-11-28T06:56:16,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00878c8327d194b967221c1933e24c9a: 2024-11-28T06:56:16,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-28T06:56:16,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:16,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 00878c8327d194b967221c1933e24c9a: 2024-11-28T06:56:16,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:16,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-28T06:56:16,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-28T06:56:16,994 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-28T06:56:16,994 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4510 sec 2024-11-28T06:56:16,998 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.4590 sec 2024-11-28T06:56:18,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-28T06:56:18,697 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:56:23,800 INFO [Thread-1709 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1709. 2024-11-28T06:56:23,800 DEBUG [Thread-1709 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:56:23,801 DEBUG [Thread-1709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:23,801 DEBUG [Thread-1709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:23,801 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:56:23,830 INFO [Thread-1715 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1715. 2024-11-28T06:56:23,830 DEBUG [Thread-1715 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:56:23,831 DEBUG [Thread-1715 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:23,831 DEBUG [Thread-1715 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:23,831 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:56:23,847 INFO [Thread-1711 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1711. 2024-11-28T06:56:23,847 DEBUG [Thread-1711 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:56:23,847 DEBUG [Thread-1711 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:23,847 DEBUG [Thread-1711 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:23,847 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:56:23,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-28T06:56:23,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-11-28T06:56:23,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 6 2024-11-28T06:56:23,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 185 2024-11-28T06:56:23,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 6 2024-11-28T06:56:23,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 177 2024-11-28T06:56:23,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-28T06:56:23,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-28T06:56:23,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1122 2024-11-28T06:56:23,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3366 rows 2024-11-28T06:56:23,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1132 2024-11-28T06:56:23,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3396 rows 2024-11-28T06:56:23,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1132 2024-11-28T06:56:23,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3391 rows 2024-11-28T06:56:23,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1128 2024-11-28T06:56:23,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3384 rows 2024-11-28T06:56:23,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1124 2024-11-28T06:56:23,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3372 rows 2024-11-28T06:56:23,848 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T06:56:23,848 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T06:56:23,848 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:23,848 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:23,848 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:56:23,849 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-28T06:56:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-28T06:56:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-28T06:56:23,852 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732776983852"}]},"ts":"1732776983852"} 2024-11-28T06:56:23,853 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-28T06:56:23,853 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-28T06:56:23,854 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T06:56:23,856 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00878c8327d194b967221c1933e24c9a, UNASSIGN}] 2024-11-28T06:56:23,857 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00878c8327d194b967221c1933e24c9a, UNASSIGN 2024-11-28T06:56:23,858 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=00878c8327d194b967221c1933e24c9a, regionState=CLOSING, regionLocation=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:23,860 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=00878c8327d194b967221c1933e24c9a, UNASSIGN because future has completed 2024-11-28T06:56:23,860 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T06:56:23,860 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; CloseRegionProcedure 00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568}] 2024-11-28T06:56:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-28T06:56:24,013 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(122): Close 00878c8327d194b967221c1933e24c9a 2024-11-28T06:56:24,013 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T06:56:24,013 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1722): Closing 00878c8327d194b967221c1933e24c9a, disabling compactions & flushes 2024-11-28T06:56:24,013 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:24,013 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:24,014 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. after waiting 0 ms 2024-11-28T06:56:24,014 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:24,014 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(2902): Flushing 00878c8327d194b967221c1933e24c9a 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-28T06:56:24,014 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=A 2024-11-28T06:56:24,015 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:24,015 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=B 2024-11-28T06:56:24,015 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:24,015 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00878c8327d194b967221c1933e24c9a, store=C 2024-11-28T06:56:24,015 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:24,023 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/3a10682d921842bfb59230f608407171 is 50, key is test_row_0/A:col10/1732776983827/Put/seqid=0 2024-11-28T06:56:24,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742269_1445 (size=9857) 2024-11-28T06:56:24,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-28T06:56:24,432 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/3a10682d921842bfb59230f608407171 2024-11-28T06:56:24,444 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/3b0ac70782be408b8a4625b553200081 is 50, key is test_row_0/B:col10/1732776983827/Put/seqid=0 2024-11-28T06:56:24,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742270_1446 (size=9857) 2024-11-28T06:56:24,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-28T06:56:24,849 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/3b0ac70782be408b8a4625b553200081 2024-11-28T06:56:24,854 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/bb5bc22dfc4d47f4942f3ac9e6e90048 is 50, key is test_row_0/C:col10/1732776983827/Put/seqid=0 2024-11-28T06:56:24,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742271_1447 (size=9857) 2024-11-28T06:56:24,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-28T06:56:25,259 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/bb5bc22dfc4d47f4942f3ac9e6e90048 2024-11-28T06:56:25,269 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/A/3a10682d921842bfb59230f608407171 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/3a10682d921842bfb59230f608407171 2024-11-28T06:56:25,272 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/3a10682d921842bfb59230f608407171, entries=100, sequenceid=463, filesize=9.6 K 2024-11-28T06:56:25,273 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/B/3b0ac70782be408b8a4625b553200081 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/3b0ac70782be408b8a4625b553200081 2024-11-28T06:56:25,276 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/3b0ac70782be408b8a4625b553200081, entries=100, sequenceid=463, filesize=9.6 K 2024-11-28T06:56:25,277 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/.tmp/C/bb5bc22dfc4d47f4942f3ac9e6e90048 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/bb5bc22dfc4d47f4942f3ac9e6e90048 2024-11-28T06:56:25,280 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/bb5bc22dfc4d47f4942f3ac9e6e90048, entries=100, sequenceid=463, filesize=9.6 K 2024-11-28T06:56:25,281 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 00878c8327d194b967221c1933e24c9a in 1267ms, sequenceid=463, compaction requested=true 2024-11-28T06:56:25,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/446d92d35d744e32b42232548450e6ee, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/76189c5406404a54be4ec05e15e65f12, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/fcff6fdb6f93468cbfc15ddb2b7a4f57, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/bab4d46746024461a8f9f0a216338e5a, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/1cbc755a7da048b99564930c168df5e4, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/c33819b0c79543eb827db5d66f79ef61, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/9bd92a9edcbb450aa02d07fec9c73713, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/2b5894fd9b9d4a9a9ad3f0c51ab30296, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/74c84ab976134dd88e28ccccf64142c8, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/89592cc4dd344b38a138308b70e144d4, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/25fd457ddde540fdac27db87629c3a07, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/2e436f5c37f847aabfa527bfcaff4107, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/882bace7a22944cc94656ef870bc9876, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/25742a8b6e09410ea7194ea294e93d22, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/9a7acb8f0d644a40907d4f736279c008, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/8aaece6975ed413ca3aa11df6ad85fd3, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/b42eed0fb8b24c74bc978901fc5688cc, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/d9cbdf578b474b8196e718f3c593c47e, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/c6a52b677f7a4ba3850c08caa14477c3, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/98d87534165e485cafe8228d4638f939, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/7b3787322e11492d848da2f784cbb9f4, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/253eaa0650da4414a9cdeb865e287f37] to archive 2024-11-28T06:56:25,282 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T06:56:25,284 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/446d92d35d744e32b42232548450e6ee to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/446d92d35d744e32b42232548450e6ee 2024-11-28T06:56:25,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/76189c5406404a54be4ec05e15e65f12 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/76189c5406404a54be4ec05e15e65f12 2024-11-28T06:56:25,287 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/fcff6fdb6f93468cbfc15ddb2b7a4f57 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/fcff6fdb6f93468cbfc15ddb2b7a4f57 2024-11-28T06:56:25,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/bab4d46746024461a8f9f0a216338e5a to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/bab4d46746024461a8f9f0a216338e5a 2024-11-28T06:56:25,290 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/1cbc755a7da048b99564930c168df5e4 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/1cbc755a7da048b99564930c168df5e4 2024-11-28T06:56:25,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/c33819b0c79543eb827db5d66f79ef61 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/c33819b0c79543eb827db5d66f79ef61 2024-11-28T06:56:25,292 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/9bd92a9edcbb450aa02d07fec9c73713 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/9bd92a9edcbb450aa02d07fec9c73713 2024-11-28T06:56:25,294 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/2b5894fd9b9d4a9a9ad3f0c51ab30296 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/2b5894fd9b9d4a9a9ad3f0c51ab30296 2024-11-28T06:56:25,295 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/74c84ab976134dd88e28ccccf64142c8 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/74c84ab976134dd88e28ccccf64142c8 2024-11-28T06:56:25,296 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/89592cc4dd344b38a138308b70e144d4 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/89592cc4dd344b38a138308b70e144d4 2024-11-28T06:56:25,298 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/25fd457ddde540fdac27db87629c3a07 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/25fd457ddde540fdac27db87629c3a07 2024-11-28T06:56:25,299 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/2e436f5c37f847aabfa527bfcaff4107 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/2e436f5c37f847aabfa527bfcaff4107 2024-11-28T06:56:25,301 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/882bace7a22944cc94656ef870bc9876 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/882bace7a22944cc94656ef870bc9876 2024-11-28T06:56:25,302 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/25742a8b6e09410ea7194ea294e93d22 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/25742a8b6e09410ea7194ea294e93d22 2024-11-28T06:56:25,303 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/9a7acb8f0d644a40907d4f736279c008 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/9a7acb8f0d644a40907d4f736279c008 2024-11-28T06:56:25,305 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/8aaece6975ed413ca3aa11df6ad85fd3 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/8aaece6975ed413ca3aa11df6ad85fd3 2024-11-28T06:56:25,306 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/b42eed0fb8b24c74bc978901fc5688cc to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/b42eed0fb8b24c74bc978901fc5688cc 2024-11-28T06:56:25,307 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/d9cbdf578b474b8196e718f3c593c47e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/d9cbdf578b474b8196e718f3c593c47e 2024-11-28T06:56:25,307 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/c6a52b677f7a4ba3850c08caa14477c3 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/c6a52b677f7a4ba3850c08caa14477c3 2024-11-28T06:56:25,308 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/98d87534165e485cafe8228d4638f939 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/98d87534165e485cafe8228d4638f939 2024-11-28T06:56:25,309 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/7b3787322e11492d848da2f784cbb9f4 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/7b3787322e11492d848da2f784cbb9f4 2024-11-28T06:56:25,310 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/253eaa0650da4414a9cdeb865e287f37 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/253eaa0650da4414a9cdeb865e287f37 2024-11-28T06:56:25,311 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/68c1bdb8b69b4971ba5fd8f93e44e5d1, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/98ea6c3fd015485c9fe06c565f2f716a, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/1855db83f0874d589175d25c20fc8519, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/67c202edc110459da488db5896fd8362, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/59cbc4ee8e594d5ea78b447c247340e5, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/aabbc387dcbb4fc9bd9301dd5f1cf86d, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/3b5cb3480dda4d3fae315fea4de1f6f6, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/bb690150437946cea82749f4a836271c, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/f3159ebb288a4f1882cbd123e039ecbe, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/7d133f98a67d488694560af4874f4957, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/72916856ee184a7aa8759edaf856ccf3, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/ea0fc35b687e409294a2ca5d4c66fd06, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/fc57924001ba47278e73aef789c8fa05, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/e6c3cddd576242ceaea0e1aa6ec1340c, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/daffdc7060524e6492d10571c59bb448, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/981802e3ca5447389ebfd37025bc4113, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/62f7acb609bf4bc483ffb76b5f2dce80, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/1538da6afe66443ca0291589b97df0ca, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/f79b58d3df104bd49778a79c2934d021, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/2d1968ec82414f75860fb4c408d05853, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/bc06cf89605d4c259a39502927f04024, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/c6b1b3325d4140659f202eac1463490c] to archive 2024-11-28T06:56:25,311 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T06:56:25,312 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/68c1bdb8b69b4971ba5fd8f93e44e5d1 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/68c1bdb8b69b4971ba5fd8f93e44e5d1 2024-11-28T06:56:25,313 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/98ea6c3fd015485c9fe06c565f2f716a to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/98ea6c3fd015485c9fe06c565f2f716a 2024-11-28T06:56:25,313 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/1855db83f0874d589175d25c20fc8519 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/1855db83f0874d589175d25c20fc8519 2024-11-28T06:56:25,314 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/67c202edc110459da488db5896fd8362 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/67c202edc110459da488db5896fd8362 2024-11-28T06:56:25,315 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/59cbc4ee8e594d5ea78b447c247340e5 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/59cbc4ee8e594d5ea78b447c247340e5 2024-11-28T06:56:25,315 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/aabbc387dcbb4fc9bd9301dd5f1cf86d to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/aabbc387dcbb4fc9bd9301dd5f1cf86d 2024-11-28T06:56:25,316 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/3b5cb3480dda4d3fae315fea4de1f6f6 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/3b5cb3480dda4d3fae315fea4de1f6f6 2024-11-28T06:56:25,317 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/bb690150437946cea82749f4a836271c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/bb690150437946cea82749f4a836271c 2024-11-28T06:56:25,318 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/f3159ebb288a4f1882cbd123e039ecbe to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/f3159ebb288a4f1882cbd123e039ecbe 2024-11-28T06:56:25,319 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/7d133f98a67d488694560af4874f4957 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/7d133f98a67d488694560af4874f4957 2024-11-28T06:56:25,320 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/72916856ee184a7aa8759edaf856ccf3 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/72916856ee184a7aa8759edaf856ccf3 2024-11-28T06:56:25,320 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/ea0fc35b687e409294a2ca5d4c66fd06 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/ea0fc35b687e409294a2ca5d4c66fd06 2024-11-28T06:56:25,321 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/fc57924001ba47278e73aef789c8fa05 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/fc57924001ba47278e73aef789c8fa05 2024-11-28T06:56:25,322 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/e6c3cddd576242ceaea0e1aa6ec1340c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/e6c3cddd576242ceaea0e1aa6ec1340c 2024-11-28T06:56:25,322 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/daffdc7060524e6492d10571c59bb448 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/daffdc7060524e6492d10571c59bb448 2024-11-28T06:56:25,323 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/981802e3ca5447389ebfd37025bc4113 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/981802e3ca5447389ebfd37025bc4113 2024-11-28T06:56:25,324 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/62f7acb609bf4bc483ffb76b5f2dce80 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/62f7acb609bf4bc483ffb76b5f2dce80 2024-11-28T06:56:25,324 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/1538da6afe66443ca0291589b97df0ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/1538da6afe66443ca0291589b97df0ca 2024-11-28T06:56:25,325 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/f79b58d3df104bd49778a79c2934d021 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/f79b58d3df104bd49778a79c2934d021 2024-11-28T06:56:25,326 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/2d1968ec82414f75860fb4c408d05853 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/2d1968ec82414f75860fb4c408d05853 2024-11-28T06:56:25,326 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/bc06cf89605d4c259a39502927f04024 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/bc06cf89605d4c259a39502927f04024 2024-11-28T06:56:25,327 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/c6b1b3325d4140659f202eac1463490c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/c6b1b3325d4140659f202eac1463490c 2024-11-28T06:56:25,328 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/4f54c3c115164b24869330622a0d3412, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/39a6248931d44eb88a87112a45e147f5, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/b3ffef4a980247dab4af259735378487, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/323df53dc3b849f0ac6ce03b6475f3a8, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/e440df1e9784413ca3180d22cbd72235, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/b731bfe1e147444a80c11af3a3e2cebd, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/0f74764310254e478a603f468db4cb1d, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/83a107ac4d7940c5af3f4906e4674b96, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/4663aebb4c4f4d9db6f69e641bb1b486, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/8ff90d93149a40e6801d3ecdf17e7621, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/b2fe3d0082f443389b26ad75ab71d200, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/3ef4d7b93e6d480da667f8c4d6a2bfd9, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/a386b050049f4bce8586bfca5b3e989e, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/e915a711d2864340b253915721e0dd89, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/c53401925fbf431fb7fb66648b940c40, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/ea0d25c782ba490097cfbbd8a6dcda3e, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/e624ba8f0e0a42219d0711d02570cd2b, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/d8b33af03b114947abc4acb47957614f, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/c1daa23b3f234f72a3718001501a487e, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/b33cb4d59bfc41e18ea2dae185ada085, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/50ac20973553435f92c77c89e576a13f, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/ec2657904ddc433187db560ce1a46609] to archive 2024-11-28T06:56:25,329 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T06:56:25,330 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/4f54c3c115164b24869330622a0d3412 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/4f54c3c115164b24869330622a0d3412 2024-11-28T06:56:25,330 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/39a6248931d44eb88a87112a45e147f5 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/39a6248931d44eb88a87112a45e147f5 2024-11-28T06:56:25,331 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/b3ffef4a980247dab4af259735378487 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/b3ffef4a980247dab4af259735378487 2024-11-28T06:56:25,332 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/323df53dc3b849f0ac6ce03b6475f3a8 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/323df53dc3b849f0ac6ce03b6475f3a8 2024-11-28T06:56:25,332 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/e440df1e9784413ca3180d22cbd72235 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/e440df1e9784413ca3180d22cbd72235 2024-11-28T06:56:25,333 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/b731bfe1e147444a80c11af3a3e2cebd to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/b731bfe1e147444a80c11af3a3e2cebd 2024-11-28T06:56:25,334 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/0f74764310254e478a603f468db4cb1d to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/0f74764310254e478a603f468db4cb1d 2024-11-28T06:56:25,334 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/83a107ac4d7940c5af3f4906e4674b96 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/83a107ac4d7940c5af3f4906e4674b96 2024-11-28T06:56:25,335 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/4663aebb4c4f4d9db6f69e641bb1b486 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/4663aebb4c4f4d9db6f69e641bb1b486 2024-11-28T06:56:25,335 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/8ff90d93149a40e6801d3ecdf17e7621 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/8ff90d93149a40e6801d3ecdf17e7621 2024-11-28T06:56:25,336 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/b2fe3d0082f443389b26ad75ab71d200 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/b2fe3d0082f443389b26ad75ab71d200 2024-11-28T06:56:25,337 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/3ef4d7b93e6d480da667f8c4d6a2bfd9 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/3ef4d7b93e6d480da667f8c4d6a2bfd9 2024-11-28T06:56:25,337 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/a386b050049f4bce8586bfca5b3e989e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/a386b050049f4bce8586bfca5b3e989e 2024-11-28T06:56:25,338 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/e915a711d2864340b253915721e0dd89 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/e915a711d2864340b253915721e0dd89 2024-11-28T06:56:25,339 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/c53401925fbf431fb7fb66648b940c40 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/c53401925fbf431fb7fb66648b940c40 2024-11-28T06:56:25,339 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/ea0d25c782ba490097cfbbd8a6dcda3e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/ea0d25c782ba490097cfbbd8a6dcda3e 2024-11-28T06:56:25,340 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/e624ba8f0e0a42219d0711d02570cd2b to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/e624ba8f0e0a42219d0711d02570cd2b 2024-11-28T06:56:25,341 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/d8b33af03b114947abc4acb47957614f to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/d8b33af03b114947abc4acb47957614f 2024-11-28T06:56:25,341 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/c1daa23b3f234f72a3718001501a487e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/c1daa23b3f234f72a3718001501a487e 2024-11-28T06:56:25,342 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/b33cb4d59bfc41e18ea2dae185ada085 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/b33cb4d59bfc41e18ea2dae185ada085 2024-11-28T06:56:25,343 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/50ac20973553435f92c77c89e576a13f to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/50ac20973553435f92c77c89e576a13f 2024-11-28T06:56:25,344 DEBUG [StoreCloser-TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/ec2657904ddc433187db560ce1a46609 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/ec2657904ddc433187db560ce1a46609 2024-11-28T06:56:25,347 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/recovered.edits/466.seqid, newMaxSeqId=466, maxSeqId=1 2024-11-28T06:56:25,348 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. 2024-11-28T06:56:25,348 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1676): Region close journal for 00878c8327d194b967221c1933e24c9a: Waiting for close lock at 1732776984013Running coprocessor pre-close hooks at 1732776984013Disabling compacts and flushes for region at 1732776984013Disabling writes for close at 1732776984014 (+1 ms)Obtaining lock to block concurrent updates at 1732776984014Preparing flush snapshotting stores in 00878c8327d194b967221c1933e24c9a at 1732776984014Finished memstore snapshotting TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732776984015 (+1 ms)Flushing stores of TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a. at 1732776984016 (+1 ms)Flushing 00878c8327d194b967221c1933e24c9a/A: creating writer at 1732776984016Flushing 00878c8327d194b967221c1933e24c9a/A: appending metadata at 1732776984022 (+6 ms)Flushing 00878c8327d194b967221c1933e24c9a/A: closing flushed file at 1732776984022Flushing 00878c8327d194b967221c1933e24c9a/B: creating writer at 1732776984441 (+419 ms)Flushing 00878c8327d194b967221c1933e24c9a/B: appending metadata at 1732776984444 (+3 ms)Flushing 00878c8327d194b967221c1933e24c9a/B: closing flushed file at 1732776984444Flushing 00878c8327d194b967221c1933e24c9a/C: creating writer at 1732776984852 (+408 ms)Flushing 00878c8327d194b967221c1933e24c9a/C: appending metadata at 1732776984854 (+2 ms)Flushing 00878c8327d194b967221c1933e24c9a/C: closing flushed file at 1732776984854Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1937d7ec: reopening flushed file at 1732776985268 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2017163: reopening flushed file at 1732776985272 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5abd948f: reopening flushed file at 1732776985276 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 00878c8327d194b967221c1933e24c9a in 1267ms, sequenceid=463, compaction requested=true at 1732776985281 (+5 ms)Writing region close event to WAL at 1732776985345 (+64 ms)Running coprocessor post-close hooks at 1732776985348 (+3 ms)Closed at 1732776985348 2024-11-28T06:56:25,349 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(157): Closed 00878c8327d194b967221c1933e24c9a 2024-11-28T06:56:25,350 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=00878c8327d194b967221c1933e24c9a, regionState=CLOSED 2024-11-28T06:56:25,351 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=130, ppid=129, state=RUNNABLE, hasLock=false; CloseRegionProcedure 00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 because future has completed 2024-11-28T06:56:25,353 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-28T06:56:25,353 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; CloseRegionProcedure 00878c8327d194b967221c1933e24c9a, server=1cb9cf437a36,43341,1732776825568 in 1.4920 sec 2024-11-28T06:56:25,354 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=129, resume processing ppid=128 2024-11-28T06:56:25,354 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, ppid=128, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00878c8327d194b967221c1933e24c9a, UNASSIGN in 1.4970 sec 2024-11-28T06:56:25,357 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-28T06:56:25,357 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5010 sec 2024-11-28T06:56:25,358 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732776985357"}]},"ts":"1732776985357"} 2024-11-28T06:56:25,359 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-28T06:56:25,359 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-28T06:56:25,360 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5100 sec 2024-11-28T06:56:25,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-28T06:56:25,997 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:56:25,998 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-28T06:56:26,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T06:56:26,001 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=131, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T06:56:26,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-28T06:56:26,003 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=131, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T06:56:26,008 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a 2024-11-28T06:56:26,011 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A, FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B, FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C, FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/recovered.edits] 2024-11-28T06:56:26,015 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/14ccd5ab960d45e9aade3f8dd74c57b3 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/14ccd5ab960d45e9aade3f8dd74c57b3 2024-11-28T06:56:26,016 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/3a10682d921842bfb59230f608407171 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/3a10682d921842bfb59230f608407171 2024-11-28T06:56:26,018 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/563ee429a0e74483bc334c0dc21d0f76 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/A/563ee429a0e74483bc334c0dc21d0f76 2024-11-28T06:56:26,021 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/3b0ac70782be408b8a4625b553200081 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/3b0ac70782be408b8a4625b553200081 2024-11-28T06:56:26,022 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/59100959a3a4409fabbf5d095a2f1640 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/59100959a3a4409fabbf5d095a2f1640 2024-11-28T06:56:26,023 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/ca9c36bf8b2d403bb30089150d1263b2 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/B/ca9c36bf8b2d403bb30089150d1263b2 2024-11-28T06:56:26,026 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/bb5bc22dfc4d47f4942f3ac9e6e90048 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/bb5bc22dfc4d47f4942f3ac9e6e90048 2024-11-28T06:56:26,027 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/f0bda250fa98442085c90311a84dd87a to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/f0bda250fa98442085c90311a84dd87a 2024-11-28T06:56:26,029 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/fa8eb46c308648ed93f91c6f73bb9113 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/C/fa8eb46c308648ed93f91c6f73bb9113 2024-11-28T06:56:26,032 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/recovered.edits/466.seqid to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a/recovered.edits/466.seqid 2024-11-28T06:56:26,032 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/00878c8327d194b967221c1933e24c9a 2024-11-28T06:56:26,033 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-28T06:56:26,035 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=131, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T06:56:26,039 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-28T06:56:26,041 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-28T06:56:26,042 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=131, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T06:56:26,042 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-28T06:56:26,043 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732776986042"}]},"ts":"9223372036854775807"} 2024-11-28T06:56:26,044 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-28T06:56:26,044 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 00878c8327d194b967221c1933e24c9a, NAME => 'TestAcidGuarantees,,1732776953119.00878c8327d194b967221c1933e24c9a.', STARTKEY => '', ENDKEY => ''}] 2024-11-28T06:56:26,045 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-28T06:56:26,045 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732776986045"}]},"ts":"9223372036854775807"} 2024-11-28T06:56:26,046 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-28T06:56:26,047 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=131, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T06:56:26,048 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 49 msec 2024-11-28T06:56:26,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-28T06:56:26,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-28T06:56:26,107 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:56:26,123 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=249 (was 250), OpenFileDescriptor=443 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=336 (was 416), ProcessCount=11 (was 11), AvailableMemoryMB=7626 (was 7810) 2024-11-28T06:56:26,132 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=249, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=336, ProcessCount=11, AvailableMemoryMB=7626 2024-11-28T06:56:26,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T06:56:26,134 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T06:56:26,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=132, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-28T06:56:26,136 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T06:56:26,136 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:26,136 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 132 2024-11-28T06:56:26,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-28T06:56:26,137 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T06:56:26,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742272_1448 (size=1061) 2024-11-28T06:56:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-28T06:56:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-28T06:56:26,548 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7 2024-11-28T06:56:26,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742273_1449 (size=53) 2024-11-28T06:56:26,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-28T06:56:26,959 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T06:56:26,959 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d1f05a2e2429924e6c4e8d32d50aa9ca, disabling compactions & flushes 2024-11-28T06:56:26,959 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:26,959 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:26,959 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. after waiting 0 ms 2024-11-28T06:56:26,959 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:26,959 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:26,959 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d1f05a2e2429924e6c4e8d32d50aa9ca: Waiting for close lock at 1732776986959Disabling compacts and flushes for region at 1732776986959Disabling writes for close at 1732776986959Writing region close event to WAL at 1732776986959Closed at 1732776986959 2024-11-28T06:56:26,991 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T06:56:26,992 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732776986992"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732776986992"}]},"ts":"1732776986992"} 2024-11-28T06:56:26,994 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T06:56:26,995 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T06:56:26,995 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732776986995"}]},"ts":"1732776986995"} 2024-11-28T06:56:26,997 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-28T06:56:26,997 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1f05a2e2429924e6c4e8d32d50aa9ca, ASSIGN}] 2024-11-28T06:56:26,999 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1f05a2e2429924e6c4e8d32d50aa9ca, ASSIGN 2024-11-28T06:56:27,000 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1f05a2e2429924e6c4e8d32d50aa9ca, ASSIGN; state=OFFLINE, location=1cb9cf437a36,43341,1732776825568; forceNewPlan=false, retain=false 2024-11-28T06:56:27,151 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=d1f05a2e2429924e6c4e8d32d50aa9ca, regionState=OPENING, regionLocation=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:27,155 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1f05a2e2429924e6c4e8d32d50aa9ca, ASSIGN because future has completed 2024-11-28T06:56:27,156 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568}] 2024-11-28T06:56:27,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-28T06:56:27,315 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:27,316 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(7752): Opening region: {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} 2024-11-28T06:56:27,316 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:27,316 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T06:56:27,317 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(7794): checking encryption for d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:27,317 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(7797): checking classloading for d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:27,319 INFO [StoreOpener-d1f05a2e2429924e6c4e8d32d50aa9ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:27,321 INFO [StoreOpener-d1f05a2e2429924e6c4e8d32d50aa9ca-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T06:56:27,322 INFO [StoreOpener-d1f05a2e2429924e6c4e8d32d50aa9ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1f05a2e2429924e6c4e8d32d50aa9ca columnFamilyName A 2024-11-28T06:56:27,322 DEBUG [StoreOpener-d1f05a2e2429924e6c4e8d32d50aa9ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:27,323 INFO [StoreOpener-d1f05a2e2429924e6c4e8d32d50aa9ca-1 {}] regionserver.HStore(327): Store=d1f05a2e2429924e6c4e8d32d50aa9ca/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:56:27,323 INFO [StoreOpener-d1f05a2e2429924e6c4e8d32d50aa9ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:27,325 INFO [StoreOpener-d1f05a2e2429924e6c4e8d32d50aa9ca-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T06:56:27,325 INFO [StoreOpener-d1f05a2e2429924e6c4e8d32d50aa9ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1f05a2e2429924e6c4e8d32d50aa9ca columnFamilyName B 2024-11-28T06:56:27,325 DEBUG [StoreOpener-d1f05a2e2429924e6c4e8d32d50aa9ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:27,326 INFO [StoreOpener-d1f05a2e2429924e6c4e8d32d50aa9ca-1 {}] regionserver.HStore(327): Store=d1f05a2e2429924e6c4e8d32d50aa9ca/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:56:27,326 INFO [StoreOpener-d1f05a2e2429924e6c4e8d32d50aa9ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:27,328 INFO [StoreOpener-d1f05a2e2429924e6c4e8d32d50aa9ca-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T06:56:27,328 INFO [StoreOpener-d1f05a2e2429924e6c4e8d32d50aa9ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1f05a2e2429924e6c4e8d32d50aa9ca columnFamilyName C 2024-11-28T06:56:27,328 DEBUG [StoreOpener-d1f05a2e2429924e6c4e8d32d50aa9ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:27,329 INFO [StoreOpener-d1f05a2e2429924e6c4e8d32d50aa9ca-1 {}] regionserver.HStore(327): Store=d1f05a2e2429924e6c4e8d32d50aa9ca/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:56:27,329 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:27,329 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1038): replaying wal for d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:27,330 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:27,330 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:27,331 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1048): stopping wal replay for d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:27,331 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1060): Cleaning up temporary data for d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:27,332 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T06:56:27,333 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1093): writing seq id for d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:27,335 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T06:56:27,336 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1114): Opened d1f05a2e2429924e6c4e8d32d50aa9ca; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67283395, jitterRate=0.0026007145643234253}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T06:56:27,336 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:27,336 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1006): Region open journal for d1f05a2e2429924e6c4e8d32d50aa9ca: Running coprocessor pre-open hook at 1732776987317Writing region info on filesystem at 1732776987317Initializing all the Stores at 1732776987318 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732776987318Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732776987319 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732776987319Cleaning up temporary data from old regions at 1732776987331 (+12 ms)Running coprocessor post-open hooks at 1732776987336 (+5 ms)Region opened successfully at 1732776987336 2024-11-28T06:56:27,337 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., pid=134, masterSystemTime=1732776987309 2024-11-28T06:56:27,339 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:27,339 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:27,340 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=d1f05a2e2429924e6c4e8d32d50aa9ca, regionState=OPEN, openSeqNum=2, regionLocation=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:27,341 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=134, ppid=133, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 because future has completed 2024-11-28T06:56:27,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-28T06:56:27,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; OpenRegionProcedure d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 in 187 msec 2024-11-28T06:56:27,348 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=132 2024-11-28T06:56:27,348 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=132, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1f05a2e2429924e6c4e8d32d50aa9ca, ASSIGN in 348 msec 2024-11-28T06:56:27,348 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T06:56:27,349 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732776987348"}]},"ts":"1732776987348"} 2024-11-28T06:56:27,350 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-28T06:56:27,351 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T06:56:27,353 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2170 sec 2024-11-28T06:56:28,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-28T06:56:28,286 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:56:28,288 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e76e6cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:28,288 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:56:28,288 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:56:28,290 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:56:28,290 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:56:28,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:56:28,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@764ea091, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:28,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:56:28,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:56:28,292 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:28,293 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50884, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:56:28,294 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fd089d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:28,295 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:56:28,296 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:56:28,297 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:56:28,298 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42142, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:56:28,299 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T06:56:28,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 1cb9cf437a36,37503,1732776824789 2024-11-28T06:56:28,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@fef0c6e 2024-11-28T06:56:28,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T06:56:28,300 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50898, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T06:56:28,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T06:56:28,301 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T06:56:28,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-28T06:56:28,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742274_1450 (size=1097) 2024-11-28T06:56:28,714 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-28T06:56:28,714 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-28T06:56:28,721 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T06:56:28,725 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1f05a2e2429924e6c4e8d32d50aa9ca, REOPEN/MOVE}] 2024-11-28T06:56:28,726 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1f05a2e2429924e6c4e8d32d50aa9ca, REOPEN/MOVE 2024-11-28T06:56:28,727 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=d1f05a2e2429924e6c4e8d32d50aa9ca, regionState=CLOSING, regionLocation=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:28,729 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1f05a2e2429924e6c4e8d32d50aa9ca, REOPEN/MOVE because future has completed 2024-11-28T06:56:28,729 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T06:56:28,729 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568}] 2024-11-28T06:56:28,883 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(122): Close d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:28,883 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T06:56:28,883 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1722): Closing d1f05a2e2429924e6c4e8d32d50aa9ca, disabling compactions & flushes 2024-11-28T06:56:28,883 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:28,883 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:28,883 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. after waiting 0 ms 2024-11-28T06:56:28,883 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:28,892 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-28T06:56:28,893 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:28,893 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1676): Region close journal for d1f05a2e2429924e6c4e8d32d50aa9ca: Waiting for close lock at 1732776988883Running coprocessor pre-close hooks at 1732776988883Disabling compacts and flushes for region at 1732776988883Disabling writes for close at 1732776988883Writing region close event to WAL at 1732776988884 (+1 ms)Running coprocessor post-close hooks at 1732776988893 (+9 ms)Closed at 1732776988893 2024-11-28T06:56:28,893 WARN [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionServer(3299): Not adding moved region record: d1f05a2e2429924e6c4e8d32d50aa9ca to self. 2024-11-28T06:56:28,896 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(157): Closed d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:28,896 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=d1f05a2e2429924e6c4e8d32d50aa9ca, regionState=CLOSED 2024-11-28T06:56:28,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 because future has completed 2024-11-28T06:56:28,902 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-28T06:56:28,902 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseRegionProcedure d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 in 170 msec 2024-11-28T06:56:28,903 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1f05a2e2429924e6c4e8d32d50aa9ca, REOPEN/MOVE; state=CLOSED, location=1cb9cf437a36,43341,1732776825568; forceNewPlan=false, retain=true 2024-11-28T06:56:29,054 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=d1f05a2e2429924e6c4e8d32d50aa9ca, regionState=OPENING, regionLocation=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:29,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1f05a2e2429924e6c4e8d32d50aa9ca, REOPEN/MOVE because future has completed 2024-11-28T06:56:29,059 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568}] 2024-11-28T06:56:29,217 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:29,218 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(7752): Opening region: {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} 2024-11-28T06:56:29,218 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:29,218 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T06:56:29,218 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(7794): checking encryption for d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:29,218 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(7797): checking classloading for d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:29,220 INFO [StoreOpener-d1f05a2e2429924e6c4e8d32d50aa9ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:29,221 INFO [StoreOpener-d1f05a2e2429924e6c4e8d32d50aa9ca-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T06:56:29,221 INFO [StoreOpener-d1f05a2e2429924e6c4e8d32d50aa9ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1f05a2e2429924e6c4e8d32d50aa9ca columnFamilyName A 2024-11-28T06:56:29,223 DEBUG [StoreOpener-d1f05a2e2429924e6c4e8d32d50aa9ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:29,224 INFO [StoreOpener-d1f05a2e2429924e6c4e8d32d50aa9ca-1 {}] regionserver.HStore(327): Store=d1f05a2e2429924e6c4e8d32d50aa9ca/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:56:29,224 INFO [StoreOpener-d1f05a2e2429924e6c4e8d32d50aa9ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:29,225 INFO [StoreOpener-d1f05a2e2429924e6c4e8d32d50aa9ca-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T06:56:29,226 INFO [StoreOpener-d1f05a2e2429924e6c4e8d32d50aa9ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1f05a2e2429924e6c4e8d32d50aa9ca columnFamilyName B 2024-11-28T06:56:29,226 DEBUG [StoreOpener-d1f05a2e2429924e6c4e8d32d50aa9ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:29,226 INFO [StoreOpener-d1f05a2e2429924e6c4e8d32d50aa9ca-1 {}] regionserver.HStore(327): Store=d1f05a2e2429924e6c4e8d32d50aa9ca/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:56:29,226 INFO [StoreOpener-d1f05a2e2429924e6c4e8d32d50aa9ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:29,227 INFO [StoreOpener-d1f05a2e2429924e6c4e8d32d50aa9ca-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T06:56:29,227 INFO [StoreOpener-d1f05a2e2429924e6c4e8d32d50aa9ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1f05a2e2429924e6c4e8d32d50aa9ca columnFamilyName C 2024-11-28T06:56:29,227 DEBUG [StoreOpener-d1f05a2e2429924e6c4e8d32d50aa9ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:29,228 INFO [StoreOpener-d1f05a2e2429924e6c4e8d32d50aa9ca-1 {}] regionserver.HStore(327): Store=d1f05a2e2429924e6c4e8d32d50aa9ca/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:56:29,228 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:29,228 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1038): replaying wal for d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:29,229 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:29,230 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:29,230 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1048): stopping wal replay for d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:29,230 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1060): Cleaning up temporary data for d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:29,231 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T06:56:29,232 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1093): writing seq id for d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:29,233 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1114): Opened d1f05a2e2429924e6c4e8d32d50aa9ca; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60816367, jitterRate=-0.0937655121088028}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T06:56:29,233 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:29,233 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1006): Region open journal for d1f05a2e2429924e6c4e8d32d50aa9ca: Running coprocessor pre-open hook at 1732776989218Writing region info on filesystem at 1732776989218Initializing all the Stores at 1732776989219 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732776989219Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732776989220 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732776989220Cleaning up temporary data from old regions at 1732776989231 (+11 ms)Running coprocessor post-open hooks at 1732776989233 (+2 ms)Region opened successfully at 1732776989233 2024-11-28T06:56:29,234 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., pid=139, masterSystemTime=1732776989213 2024-11-28T06:56:29,235 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:29,235 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:29,236 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=d1f05a2e2429924e6c4e8d32d50aa9ca, regionState=OPEN, openSeqNum=5, regionLocation=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:29,237 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 because future has completed 2024-11-28T06:56:29,239 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=137 2024-11-28T06:56:29,239 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=137, state=SUCCESS, hasLock=false; OpenRegionProcedure d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 in 180 msec 2024-11-28T06:56:29,240 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-28T06:56:29,240 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1f05a2e2429924e6c4e8d32d50aa9ca, REOPEN/MOVE in 514 msec 2024-11-28T06:56:29,242 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-28T06:56:29,242 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 521 msec 2024-11-28T06:56:29,244 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 941 msec 2024-11-28T06:56:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-28T06:56:29,245 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:56:29,245 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2aa61c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:29,245 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:56:29,245 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:56:29,246 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:56:29,246 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:56:29,246 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:56:29,246 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c4bc217, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:29,246 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:56:29,246 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:56:29,246 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:29,247 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50914, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:56:29,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14b2922a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:29,248 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca6c80d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:29,248 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:56:29,248 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:56:29,248 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:56:29,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:56:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:56:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@536c4b2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:56:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:56:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:29,250 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50922, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:56:29,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@341f18fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:29,250 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14afc90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:29,251 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:56:29,251 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:56:29,251 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:56:29,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:56:29,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:56:29,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c5c26f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:29,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:56:29,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:56:29,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:29,252 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:56:29,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b3021e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:29,253 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb57557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:29,253 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:56:29,253 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:56:29,254 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:56:29,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:56:29,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:56:29,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29a6f006, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:29,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:56:29,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:56:29,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:29,255 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50974, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:56:29,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c0e5ea5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:29,256 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e0b6ba1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:29,256 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:56:29,256 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:56:29,256 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:56:29,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:56:29,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:56:29,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60d896b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:29,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:56:29,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:56:29,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:29,258 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51000, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:56:29,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@286ca41b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:29,259 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af69a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:29,259 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:56:29,259 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:56:29,259 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:56:29,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:56:29,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:56:29,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dc4e749, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:29,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:56:29,260 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:56:29,260 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:29,260 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51028, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:56:29,261 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e9628b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:29,261 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd90259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:29,261 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:56:29,261 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:56:29,262 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:56:29,262 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:56:29,262 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:56:29,262 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70ac974a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:29,262 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:56:29,262 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:56:29,262 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:29,263 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51042, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:56:29,263 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d71fa04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:29,264 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594d5ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:29,264 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:56:29,264 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:56:29,265 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:56:29,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:56:29,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:56:29,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1356892e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:29,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:56:29,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:56:29,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:29,266 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51058, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:56:29,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59bf7111, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:29,267 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267cfa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:29,268 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:56:29,268 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:56:29,268 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:56:29,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:56:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:56:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1293f2f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:56:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:56:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:29,270 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51072, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:56:29,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a03f0cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:29,271 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c58aaf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:29,271 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,37503,-1 for getting cluster id 2024-11-28T06:56:29,271 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:56:29,272 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2033d23-da74-47d8-82e3-5e5da01454bd' 2024-11-28T06:56:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:56:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2033d23-da74-47d8-82e3-5e5da01454bd" 2024-11-28T06:56:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15d81bf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,37503,-1] 2024-11-28T06:56:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:56:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:29,273 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51088, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:56:29,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10a99139, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:29,274 DEBUG [Thread-2064 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:56:29,274 DEBUG [Thread-2068 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:56:29,274 DEBUG [Thread-2062 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:56:29,274 DEBUG [Thread-2057 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:56:29,274 DEBUG [Thread-2053 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:56:29,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:56:29,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:56:29,275 DEBUG [Thread-2066 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:56:29,275 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:56:29,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:56:29,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:56:29,275 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:56:29,275 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:56:29,275 DEBUG [Thread-2059 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:56:29,276 DEBUG [Thread-2055 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:56:29,276 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42164, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:56:29,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:56:29,276 DEBUG [Thread-2066 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:56:29,276 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42156, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:56:29,276 DEBUG [Thread-2070 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:56:29,276 DEBUG [Thread-2066 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:56:29,277 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:56:29,277 DEBUG [Thread-2053 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:56:29,277 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42162, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:56:29,277 DEBUG [Thread-2053 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:56:29,277 DEBUG [Thread-2055 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:56:29,277 DEBUG [Thread-2055 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:56:29,277 DEBUG [Thread-2051 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:56:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees 2024-11-28T06:56:29,278 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42186, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:56:29,278 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42192, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:56:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:56:29,278 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42204, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:56:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:56:29,279 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:56:29,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:56:29,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5] 2024-11-28T06:56:29,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,43341,1732776825568, seqNum=-1] 2024-11-28T06:56:29,279 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:56:29,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:56:29,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:56:29,280 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42208, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:56:29,280 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42230, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:56:29,281 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:56:29,281 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42240, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:56:29,281 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:56:29,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5] 2024-11-28T06:56:29,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5] 2024-11-28T06:56:29,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5] 2024-11-28T06:56:29,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5] 2024-11-28T06:56:29,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-28T06:56:29,283 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5] 2024-11-28T06:56:29,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5] 2024-11-28T06:56:29,284 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5] 2024-11-28T06:56:29,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5] 2024-11-28T06:56:29,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5] 2024-11-28T06:56:29,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:29,292 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1f05a2e2429924e6c4e8d32d50aa9ca 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T06:56:29,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=A 2024-11-28T06:56:29,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:29,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=B 2024-11-28T06:56:29,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:29,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=C 2024-11-28T06:56:29,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:29,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42208 deadline: 1732777049309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:29,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:29,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777049311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:29,314 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,314 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,314 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:29,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42230 deadline: 1732777049312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:29,315 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:29,315 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,315 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42204 deadline: 1732777049313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:29,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:29,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777049315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:29,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:29,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411283eaff75ddeb54d0bb39551c124d56c4d_d1f05a2e2429924e6c4e8d32d50aa9ca is 50, key is test_row_0/A:col10/1732776989290/Put/seqid=0 2024-11-28T06:56:29,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742275_1451 (size=12154) 2024-11-28T06:56:29,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-28T06:56:29,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:29,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42208 deadline: 1732777049415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:29,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:29,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:29,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777049415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:29,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:29,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:29,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42230 deadline: 1732777049425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:29,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:29,428 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42204 deadline: 1732777049426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:29,428 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,428 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:29,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:29,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:29,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777049426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:29,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:29,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-28T06:56:29,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:29,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:29,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:29,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:29,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:29,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:29,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-28T06:56:29,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:29,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:29,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:29,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:29,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:29,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:29,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-28T06:56:29,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42208 deadline: 1732777049625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:29,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:29,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777049625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:29,628 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,628 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,628 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:29,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:29,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42230 deadline: 1732777049636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:29,638 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:29,638 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,638 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:29,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42204 deadline: 1732777049636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:29,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:29,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:29,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777049636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:29,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:29,724 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:29,727 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411283eaff75ddeb54d0bb39551c124d56c4d_d1f05a2e2429924e6c4e8d32d50aa9ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411283eaff75ddeb54d0bb39551c124d56c4d_d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:29,728 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/421b9da479de45efa00010683b0a2ca3, store: [table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:29,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/421b9da479de45efa00010683b0a2ca3 is 175, key is test_row_0/A:col10/1732776989290/Put/seqid=0 2024-11-28T06:56:29,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742276_1452 (size=30955) 2024-11-28T06:56:29,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-28T06:56:29,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:29,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:29,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:29,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:29,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:29,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:29,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-28T06:56:29,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:29,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:29,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:29,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:29,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:29,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:29,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-28T06:56:29,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:29,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42208 deadline: 1732777049935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:29,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:29,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777049935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:29,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:29,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:29,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:29,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42230 deadline: 1732777049946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:29,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:29,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42204 deadline: 1732777049946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:29,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:29,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:29,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777049946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:29,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:29,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:29,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:30,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-28T06:56:30,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:30,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:30,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:30,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:30,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:30,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:30,133 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/421b9da479de45efa00010683b0a2ca3 2024-11-28T06:56:30,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/b4e6362f3bf0453b9b50ed6981103ce6 is 50, key is test_row_0/B:col10/1732776989290/Put/seqid=0 2024-11-28T06:56:30,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742277_1453 (size=12001) 2024-11-28T06:56:30,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-28T06:56:30,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:30,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:30,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:30,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:30,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:30,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:30,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-28T06:56:30,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:30,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:30,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:30,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:30,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:30,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:30,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-28T06:56:30,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:30,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42208 deadline: 1732777050446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:30,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:30,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:30,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:30,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:30,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777050446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:30,449 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:30,449 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:30,450 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:30,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:30,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42230 deadline: 1732777050455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:30,459 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:30,459 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:30,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:30,459 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:30,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42204 deadline: 1732777050455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:30,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:30,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777050455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:30,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:30,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:30,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:30,459 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:30,459 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:30,459 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:30,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-28T06:56:30,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:30,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:30,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:30,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:30,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:30,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:30,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/b4e6362f3bf0453b9b50ed6981103ce6 2024-11-28T06:56:30,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/2f9f93c04e4f4422bf02431cc6099711 is 50, key is test_row_0/C:col10/1732776989290/Put/seqid=0 2024-11-28T06:56:30,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742278_1454 (size=12001) 2024-11-28T06:56:30,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/2f9f93c04e4f4422bf02431cc6099711 2024-11-28T06:56:30,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/421b9da479de45efa00010683b0a2ca3 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/421b9da479de45efa00010683b0a2ca3 2024-11-28T06:56:30,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/421b9da479de45efa00010683b0a2ca3, entries=150, sequenceid=16, filesize=30.2 K 2024-11-28T06:56:30,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/b4e6362f3bf0453b9b50ed6981103ce6 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/b4e6362f3bf0453b9b50ed6981103ce6 2024-11-28T06:56:30,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/b4e6362f3bf0453b9b50ed6981103ce6, entries=150, sequenceid=16, filesize=11.7 K 2024-11-28T06:56:30,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/2f9f93c04e4f4422bf02431cc6099711 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/2f9f93c04e4f4422bf02431cc6099711 2024-11-28T06:56:30,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/2f9f93c04e4f4422bf02431cc6099711, entries=150, sequenceid=16, filesize=11.7 K 2024-11-28T06:56:30,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for d1f05a2e2429924e6c4e8d32d50aa9ca in 1307ms, sequenceid=16, compaction requested=false 2024-11-28T06:56:30,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:30,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-28T06:56:30,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:30,659 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2902): Flushing d1f05a2e2429924e6c4e8d32d50aa9ca 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T06:56:30,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=A 2024-11-28T06:56:30,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:30,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=B 2024-11-28T06:56:30,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:30,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=C 2024-11-28T06:56:30,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:30,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411287b2f905c69f7443098772d5b2a6ca112_d1f05a2e2429924e6c4e8d32d50aa9ca is 50, key is test_row_0/A:col10/1732776989312/Put/seqid=0 2024-11-28T06:56:30,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742279_1455 (size=12154) 2024-11-28T06:56:30,872 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T06:56:31,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:31,076 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411287b2f905c69f7443098772d5b2a6ca112_d1f05a2e2429924e6c4e8d32d50aa9ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411287b2f905c69f7443098772d5b2a6ca112_d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:31,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/57d6480fea0b499497eaa7fe37479afb, store: [table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:31,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/57d6480fea0b499497eaa7fe37479afb is 175, key is test_row_0/A:col10/1732776989312/Put/seqid=0 2024-11-28T06:56:31,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742280_1456 (size=30955) 2024-11-28T06:56:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-28T06:56:31,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:31,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:31,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:31,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42230 deadline: 1732777051476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42204 deadline: 1732777051476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:31,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42208 deadline: 1732777051476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:31,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:31,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:31,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:31,479 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_2' in d1f05a2e2429924e6c4e8d32d50aa9ca of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:56:31,479 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_2' in d1f05a2e2429924e6c4e8d32d50aa9ca of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:56:31,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:31,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:31,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:31,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:31,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:31,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:31,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777051478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:31,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777051479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:31,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:31,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:31,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:31,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:31,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:31,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:31,502 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/57d6480fea0b499497eaa7fe37479afb 2024-11-28T06:56:31,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/b594710925724c028c3cd11780a38a2b is 50, key is test_row_0/B:col10/1732776989312/Put/seqid=0 2024-11-28T06:56:31,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742281_1457 (size=12001) 2024-11-28T06:56:31,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42208 deadline: 1732777051585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:31,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777051585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:31,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777051585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:31,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:31,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42208 deadline: 1732777051796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:31,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:31,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:31,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:31,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777051796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:31,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:31,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:31,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:31,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:31,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:31,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777051796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:31,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:31,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:31,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:31,911 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/b594710925724c028c3cd11780a38a2b 2024-11-28T06:56:31,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/c4b1eddabe9242ba97a1c9e37275e4cf is 50, key is test_row_0/C:col10/1732776989312/Put/seqid=0 2024-11-28T06:56:31,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742282_1458 (size=12001) 2024-11-28T06:56:31,920 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/c4b1eddabe9242ba97a1c9e37275e4cf 2024-11-28T06:56:31,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/57d6480fea0b499497eaa7fe37479afb as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/57d6480fea0b499497eaa7fe37479afb 2024-11-28T06:56:31,926 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/57d6480fea0b499497eaa7fe37479afb, entries=150, sequenceid=36, filesize=30.2 K 2024-11-28T06:56:31,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/b594710925724c028c3cd11780a38a2b as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/b594710925724c028c3cd11780a38a2b 2024-11-28T06:56:31,929 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/b594710925724c028c3cd11780a38a2b, entries=150, sequenceid=36, filesize=11.7 K 2024-11-28T06:56:31,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/c4b1eddabe9242ba97a1c9e37275e4cf as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/c4b1eddabe9242ba97a1c9e37275e4cf 2024-11-28T06:56:31,941 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/c4b1eddabe9242ba97a1c9e37275e4cf, entries=150, sequenceid=36, filesize=11.7 K 2024-11-28T06:56:31,942 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for d1f05a2e2429924e6c4e8d32d50aa9ca in 1282ms, sequenceid=36, compaction requested=false 2024-11-28T06:56:31,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2603): Flush status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:31,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:31,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=141 2024-11-28T06:56:31,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=141 2024-11-28T06:56:31,944 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-28T06:56:31,944 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6620 sec 2024-11-28T06:56:31,946 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees in 2.6700 sec 2024-11-28T06:56:32,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:32,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1f05a2e2429924e6c4e8d32d50aa9ca 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T06:56:32,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=A 2024-11-28T06:56:32,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:32,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=B 2024-11-28T06:56:32,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:32,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=C 2024-11-28T06:56:32,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:32,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128e00558b28c0344cc98547134769333ec_d1f05a2e2429924e6c4e8d32d50aa9ca is 50, key is test_row_0/A:col10/1732776992107/Put/seqid=0 2024-11-28T06:56:32,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:32,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42208 deadline: 1732777052135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:32,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:32,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:32,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:32,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742283_1459 (size=14594) 2024-11-28T06:56:32,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:32,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:32,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777052142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:32,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777052142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:32,148 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:32,148 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:32,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:32,148 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:32,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:32,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:32,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:32,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42208 deadline: 1732777052246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:32,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:32,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:32,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:32,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:32,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777052256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:32,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:32,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:32,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777052256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:32,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:32,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:32,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:32,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:32,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:32,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:32,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42208 deadline: 1732777052455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:32,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:32,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:32,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:32,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:32,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777052466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:32,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:32,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777052466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:32,469 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:32,469 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:32,469 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:32,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:32,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:32,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:32,544 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:32,547 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128e00558b28c0344cc98547134769333ec_d1f05a2e2429924e6c4e8d32d50aa9ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e00558b28c0344cc98547134769333ec_d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:32,548 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/a662c96cab5d4c8eb97aaba5ba16cf46, store: [table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:32,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/a662c96cab5d4c8eb97aaba5ba16cf46 is 175, key is test_row_0/A:col10/1732776992107/Put/seqid=0 2024-11-28T06:56:32,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742284_1460 (size=39549) 2024-11-28T06:56:32,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:32,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42208 deadline: 1732777052765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:32,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:32,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:32,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:32,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:32,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777052775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:32,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:32,777 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:32,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777052775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:32,777 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:32,777 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:32,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:32,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:32,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:32,952 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/a662c96cab5d4c8eb97aaba5ba16cf46 2024-11-28T06:56:32,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/8f6fc64673ea4c92a8a82af15c50142e is 50, key is test_row_0/B:col10/1732776992107/Put/seqid=0 2024-11-28T06:56:32,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742285_1461 (size=12001) 2024-11-28T06:56:33,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:33,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42208 deadline: 1732777053276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:33,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:33,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:33,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:33,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:33,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777053285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:33,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:33,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:33,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:33,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:33,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777053285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:33,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/8f6fc64673ea4c92a8a82af15c50142e 2024-11-28T06:56:33,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/30069877e342437f90c04bc58dbaf500 is 50, key is test_row_0/C:col10/1732776992107/Put/seqid=0 2024-11-28T06:56:33,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742286_1462 (size=12001) 2024-11-28T06:56:33,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/30069877e342437f90c04bc58dbaf500 2024-11-28T06:56:33,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/a662c96cab5d4c8eb97aaba5ba16cf46 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/a662c96cab5d4c8eb97aaba5ba16cf46 2024-11-28T06:56:33,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/a662c96cab5d4c8eb97aaba5ba16cf46, entries=200, sequenceid=49, filesize=38.6 K 2024-11-28T06:56:33,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/8f6fc64673ea4c92a8a82af15c50142e as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/8f6fc64673ea4c92a8a82af15c50142e 2024-11-28T06:56:33,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/8f6fc64673ea4c92a8a82af15c50142e, entries=150, sequenceid=49, filesize=11.7 K 2024-11-28T06:56:33,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/30069877e342437f90c04bc58dbaf500 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/30069877e342437f90c04bc58dbaf500 2024-11-28T06:56:33,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/30069877e342437f90c04bc58dbaf500, entries=150, sequenceid=49, filesize=11.7 K 2024-11-28T06:56:33,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for d1f05a2e2429924e6c4e8d32d50aa9ca in 1281ms, sequenceid=49, compaction requested=true 2024-11-28T06:56:33,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:33,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f05a2e2429924e6c4e8d32d50aa9ca:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:56:33,389 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:56:33,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:33,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f05a2e2429924e6c4e8d32d50aa9ca:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:56:33,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:33,389 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:56:33,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f05a2e2429924e6c4e8d32d50aa9ca:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:56:33,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:56:33,390 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:56:33,390 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:56:33,390 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): d1f05a2e2429924e6c4e8d32d50aa9ca/B is initiating minor compaction (all files) 2024-11-28T06:56:33,390 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): d1f05a2e2429924e6c4e8d32d50aa9ca/A is initiating minor compaction (all files) 2024-11-28T06:56:33,390 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f05a2e2429924e6c4e8d32d50aa9ca/A in TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:33,390 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f05a2e2429924e6c4e8d32d50aa9ca/B in TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:33,390 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/421b9da479de45efa00010683b0a2ca3, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/57d6480fea0b499497eaa7fe37479afb, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/a662c96cab5d4c8eb97aaba5ba16cf46] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp, totalSize=99.1 K 2024-11-28T06:56:33,390 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/b4e6362f3bf0453b9b50ed6981103ce6, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/b594710925724c028c3cd11780a38a2b, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/8f6fc64673ea4c92a8a82af15c50142e] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp, totalSize=35.2 K 2024-11-28T06:56:33,390 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:33,390 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. files: [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/421b9da479de45efa00010683b0a2ca3, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/57d6480fea0b499497eaa7fe37479afb, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/a662c96cab5d4c8eb97aaba5ba16cf46] 2024-11-28T06:56:33,390 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 421b9da479de45efa00010683b0a2ca3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732776989283 2024-11-28T06:56:33,390 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting b4e6362f3bf0453b9b50ed6981103ce6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732776989283 2024-11-28T06:56:33,390 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57d6480fea0b499497eaa7fe37479afb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732776989302 2024-11-28T06:56:33,390 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting b594710925724c028c3cd11780a38a2b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732776989302 2024-11-28T06:56:33,390 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting a662c96cab5d4c8eb97aaba5ba16cf46, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732776991466 2024-11-28T06:56:33,390 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f6fc64673ea4c92a8a82af15c50142e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732776991466 2024-11-28T06:56:33,398 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:33,400 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f05a2e2429924e6c4e8d32d50aa9ca#B#compaction#391 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:33,400 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/f89b89d6c42a433a8a74ee42ca16b94e is 50, key is test_row_0/B:col10/1732776992107/Put/seqid=0 2024-11-28T06:56:33,403 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128f92f93053bc2482b86794380459a41ed_d1f05a2e2429924e6c4e8d32d50aa9ca store=[table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:33,405 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128f92f93053bc2482b86794380459a41ed_d1f05a2e2429924e6c4e8d32d50aa9ca, store=[table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:33,405 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128f92f93053bc2482b86794380459a41ed_d1f05a2e2429924e6c4e8d32d50aa9ca because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:33,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742287_1463 (size=12104) 2024-11-28T06:56:33,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742288_1464 (size=4469) 2024-11-28T06:56:33,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-28T06:56:33,446 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:56:33,446 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:56:33,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2024-11-28T06:56:33,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-28T06:56:33,449 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:56:33,449 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:56:33,449 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:56:33,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:33,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732776992132, max timestamp=1732776993497Num uniques 859; 2024-11-28T06:56:33,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T06:56:33,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732776992132, max timestamp=1732776993497Num uniques 859; 2024-11-28T06:56:33,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732776992132, max timestamp=1732776993497Num uniques 859; 2024-11-28T06:56:33,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T06:56:33,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T06:56:33,501 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1f05a2e2429924e6c4e8d32d50aa9ca 3/3 column families, dataSize=26.84 KB heapSize=66.23 KB 2024-11-28T06:56:33,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=A 2024-11-28T06:56:33,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:56:33,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=B 2024-11-28T06:56:33,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:56:33,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=C 2024-11-28T06:56:33,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:56:33,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411282070e480c71041738114c6184a0ca977_d1f05a2e2429924e6c4e8d32d50aa9ca is 50, key is test_row_0/A:col10/1732776992140/Put/seqid=0 2024-11-28T06:56:33,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742289_1465 (size=12154) 2024-11-28T06:56:33,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-28T06:56:33,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732776993543, max timestamp=1732776993555Num uniques 859; 2024-11-28T06:56:33,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T06:56:33,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732776993543, max timestamp=1732776993555Num uniques 859; 2024-11-28T06:56:33,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732776993543, max timestamp=1732776993555Num uniques 859; 2024-11-28T06:56:33,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T06:56:33,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T06:56:33,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-28T06:56:33,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:33,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:33,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:33,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:33,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:33,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:33,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732776993592, max timestamp=1732776993615Num uniques 1018; 2024-11-28T06:56:33,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732776993592, max timestamp=1732776993615Num uniques 1018; 2024-11-28T06:56:33,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-28T06:56:33,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-28T06:56:33,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732776993592, max timestamp=1732776993615Num uniques 1018; 2024-11-28T06:56:33,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-28T06:56:33,676 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732776993648, max timestamp=1732776993672Num uniques 1027; 2024-11-28T06:56:33,676 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-28T06:56:33,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732776993648, max timestamp=1732776993672Num uniques 1027; 2024-11-28T06:56:33,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732776993648, max timestamp=1732776993672Num uniques 1027; 2024-11-28T06:56:33,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-28T06:56:33,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-28T06:56:33,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732776993690, max timestamp=1732776993722Num uniques 1036; 2024-11-28T06:56:33,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732776993690, max timestamp=1732776993722Num uniques 1036; 2024-11-28T06:56:33,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-28T06:56:33,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-28T06:56:33,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732776993690, max timestamp=1732776993722Num uniques 1036; 2024-11-28T06:56:33,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-28T06:56:33,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-28T06:56:33,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:33,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:33,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:33,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:33,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:33,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:33,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-28T06:56:33,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732776993755, max timestamp=1732776993800Num uniques 1095; 2024-11-28T06:56:33,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-28T06:56:33,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732776993755, max timestamp=1732776993800Num uniques 1095; 2024-11-28T06:56:33,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-28T06:56:33,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732776993755, max timestamp=1732776993800Num uniques 1095; 2024-11-28T06:56:33,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-28T06:56:33,813 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f05a2e2429924e6c4e8d32d50aa9ca#A#compaction#392 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:33,813 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/6838b68679db4f309cd6da111982fe0d is 175, key is test_row_0/A:col10/1732776992107/Put/seqid=0 2024-11-28T06:56:33,815 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/f89b89d6c42a433a8a74ee42ca16b94e as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/f89b89d6c42a433a8a74ee42ca16b94e 2024-11-28T06:56:33,820 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f05a2e2429924e6c4e8d32d50aa9ca/B of d1f05a2e2429924e6c4e8d32d50aa9ca into f89b89d6c42a433a8a74ee42ca16b94e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:33,820 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:33,820 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., storeName=d1f05a2e2429924e6c4e8d32d50aa9ca/B, priority=13, startTime=1732776993389; duration=0sec 2024-11-28T06:56:33,820 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:56:33,821 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f05a2e2429924e6c4e8d32d50aa9ca:B 2024-11-28T06:56:33,821 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:56:33,821 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:56:33,821 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): d1f05a2e2429924e6c4e8d32d50aa9ca/C is initiating minor compaction (all files) 2024-11-28T06:56:33,821 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f05a2e2429924e6c4e8d32d50aa9ca/C in TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:33,822 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/2f9f93c04e4f4422bf02431cc6099711, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/c4b1eddabe9242ba97a1c9e37275e4cf, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/30069877e342437f90c04bc58dbaf500] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp, totalSize=35.2 K 2024-11-28T06:56:33,822 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f9f93c04e4f4422bf02431cc6099711, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732776989283 2024-11-28T06:56:33,823 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting c4b1eddabe9242ba97a1c9e37275e4cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732776989302 2024-11-28T06:56:33,823 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 30069877e342437f90c04bc58dbaf500, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732776991466 2024-11-28T06:56:33,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742290_1466 (size=31058) 2024-11-28T06:56:33,844 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f05a2e2429924e6c4e8d32d50aa9ca#C#compaction#394 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:33,845 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/8d213d83924943e1b3b5d15fe352d67a is 50, key is test_row_0/C:col10/1732776992107/Put/seqid=0 2024-11-28T06:56:33,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742291_1467 (size=12104) 2024-11-28T06:56:33,861 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/8d213d83924943e1b3b5d15fe352d67a as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/8d213d83924943e1b3b5d15fe352d67a 2024-11-28T06:56:33,867 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f05a2e2429924e6c4e8d32d50aa9ca/C of d1f05a2e2429924e6c4e8d32d50aa9ca into 8d213d83924943e1b3b5d15fe352d67a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:33,867 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:33,867 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., storeName=d1f05a2e2429924e6c4e8d32d50aa9ca/C, priority=13, startTime=1732776993389; duration=0sec 2024-11-28T06:56:33,867 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:33,867 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f05a2e2429924e6c4e8d32d50aa9ca:C 2024-11-28T06:56:33,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732776993843, max timestamp=1732776993876Num uniques 1104; 2024-11-28T06:56:33,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732776993843, max timestamp=1732776993876Num uniques 1104; 2024-11-28T06:56:33,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-28T06:56:33,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-28T06:56:33,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732776993843, max timestamp=1732776993876Num uniques 1104; 2024-11-28T06:56:33,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-28T06:56:33,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-28T06:56:33,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:33,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:33,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:33,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:33,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:33,913 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:33,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:33,916 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411282070e480c71041738114c6184a0ca977_d1f05a2e2429924e6c4e8d32d50aa9ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411282070e480c71041738114c6184a0ca977_d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:33,917 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/899e47f9a9c14af5b1860932d68af59e, store: [table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:33,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/899e47f9a9c14af5b1860932d68af59e is 175, key is test_row_0/A:col10/1732776992140/Put/seqid=0 2024-11-28T06:56:33,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742292_1468 (size=30955) 2024-11-28T06:56:33,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732776993906, max timestamp=1732776993951Num uniques 1063; 2024-11-28T06:56:33,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732776993906, max timestamp=1732776993951Num uniques 1063; 2024-11-28T06:56:33,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732776993906, max timestamp=1732776993951Num uniques 1063; 2024-11-28T06:56:33,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-28T06:56:33,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-28T06:56:33,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-28T06:56:34,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1732776993977, max timestamp=1732776994015Num uniques 1022; 2024-11-28T06:56:34,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-28T06:56:34,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1732776993977, max timestamp=1732776994015Num uniques 1022; 2024-11-28T06:56:34,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1732776993977, max timestamp=1732776994015Num uniques 1022; 2024-11-28T06:56:34,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-28T06:56:34,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-28T06:56:34,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-28T06:56:34,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:34,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:34,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:34,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:34,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:34,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:34,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-28T06:56:34,077 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1732776994057, max timestamp=1732776994073Num uniques 1081; 2024-11-28T06:56:34,077 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-28T06:56:34,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8285, totalHeapSize=19064, min timestamp=1732776994057, max timestamp=1732776994073Num uniques 1081; 2024-11-28T06:56:34,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8280, totalHeapSize=19064, min timestamp=1732776994057, max timestamp=1732776994073Num uniques 1081; 2024-11-28T06:56:34,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49505, new segment data size=8285, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-28T06:56:34,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49500, new segment data size=8280, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-28T06:56:34,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1732776994108, max timestamp=1732776994119Num uniques 1040; 2024-11-28T06:56:34,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1732776994108, max timestamp=1732776994119Num uniques 1040; 2024-11-28T06:56:34,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-11-28T06:56:34,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-11-28T06:56:34,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1732776994108, max timestamp=1732776994119Num uniques 1040; 2024-11-28T06:56:34,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-11-28T06:56:34,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1732776994143, max timestamp=1732776994167Num uniques 1049; 2024-11-28T06:56:34,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=11404, suffix heap size=137064,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=249 2024-11-28T06:56:34,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1732776994143, max timestamp=1732776994167Num uniques 1049; 2024-11-28T06:56:34,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=11404, suffix heap size=137064,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=249 2024-11-28T06:56:34,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1732776994143, max timestamp=1732776994167Num uniques 1049; 2024-11-28T06:56:34,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-11-28T06:56:34,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-28T06:56:34,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:34,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:34,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:34,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:34,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:34,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:34,223 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1732776994205, max timestamp=1732776994218Num uniques 1108; 2024-11-28T06:56:34,223 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1732776994205, max timestamp=1732776994218Num uniques 1058; 2024-11-28T06:56:34,223 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1732776994205, max timestamp=1732776994218Num uniques 1108; 2024-11-28T06:56:34,223 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=9520, suffix heap size=143200,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=208 2024-11-28T06:56:34,224 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=9520, suffix heap size=138000,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=208 2024-11-28T06:56:34,224 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=9520, suffix heap size=143200,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=208 2024-11-28T06:56:34,230 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/6838b68679db4f309cd6da111982fe0d as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/6838b68679db4f309cd6da111982fe0d 2024-11-28T06:56:34,234 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f05a2e2429924e6c4e8d32d50aa9ca/A of d1f05a2e2429924e6c4e8d32d50aa9ca into 6838b68679db4f309cd6da111982fe0d(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:34,234 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:34,234 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., storeName=d1f05a2e2429924e6c4e8d32d50aa9ca/A, priority=13, startTime=1732776993389; duration=0sec 2024-11-28T06:56:34,234 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:34,234 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f05a2e2429924e6c4e8d32d50aa9ca:A 2024-11-28T06:56:34,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1732776994243, max timestamp=1732776994264Num uniques 1067; 2024-11-28T06:56:34,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=9932, suffix heap size=138936,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=217 2024-11-28T06:56:34,268 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1732776994243, max timestamp=1732776994264Num uniques 1067; 2024-11-28T06:56:34,268 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=9932, suffix heap size=138936,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=217 2024-11-28T06:56:34,268 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1732776994243, max timestamp=1732776994264Num uniques 1067; 2024-11-28T06:56:34,268 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=9932, suffix heap size=138936,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=217 2024-11-28T06:56:34,321 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=70, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/899e47f9a9c14af5b1860932d68af59e 2024-11-28T06:56:34,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/2419ee7c3e424462bfeaf81814ddadf5 is 50, key is test_row_0/B:col10/1732776992140/Put/seqid=0 2024-11-28T06:56:34,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742293_1469 (size=12001) 2024-11-28T06:56:34,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10336, totalHeapSize=23744, min timestamp=1732776994292, max timestamp=1732776994303Num uniques 1076; 2024-11-28T06:56:34,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49266, new segment data size=10336, suffix heap size=139872,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=226 2024-11-28T06:56:34,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10336, totalHeapSize=23744, min timestamp=1732776994292, max timestamp=1732776994303Num uniques 1076; 2024-11-28T06:56:34,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49266, new segment data size=10336, suffix heap size=139872,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=226 2024-11-28T06:56:34,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1732776994292, max timestamp=1732776994303Num uniques 1076; 2024-11-28T06:56:34,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=10346, suffix heap size=139872,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=226 2024-11-28T06:56:34,358 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=285, cellSize=13050, totalHeapSize=29880, min timestamp=1732776994345, max timestamp=1732776994354Num uniques 1085; 2024-11-28T06:56:34,358 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=285, cellSize=13050, totalHeapSize=29880, min timestamp=1732776994345, max timestamp=1732776994354Num uniques 1085; 2024-11-28T06:56:34,358 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=285, cellSize=13050, totalHeapSize=29880, min timestamp=1732776994345, max timestamp=1732776994354Num uniques 1085; 2024-11-28T06:56:34,358 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=13050, suffix heap size=140808,new segment heap size=29880 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=285 2024-11-28T06:56:34,358 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=13050, suffix heap size=140808,new segment heap size=29880 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=285 2024-11-28T06:56:34,358 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=13050, suffix heap size=140808,new segment heap size=29880 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=285 2024-11-28T06:56:34,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-28T06:56:34,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:34,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:34,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:34,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:34,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:34,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:34,388 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8884, totalHeapSize=20416, min timestamp=1732776994379, max timestamp=1732776994384Num uniques 1144; 2024-11-28T06:56:34,388 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8884, totalHeapSize=20416, min timestamp=1732776994379, max timestamp=1732776994384Num uniques 1144; 2024-11-28T06:56:34,388 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52394, new segment data size=8884, suffix heap size=146944,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1144, new segment cells count=194 2024-11-28T06:56:34,388 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52394, new segment data size=8884, suffix heap size=146944,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1144, new segment cells count=194 2024-11-28T06:56:34,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8884, totalHeapSize=20416, min timestamp=1732776994379, max timestamp=1732776994384Num uniques 1144; 2024-11-28T06:56:34,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52394, new segment data size=8884, suffix heap size=146944,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1144, new segment cells count=194 2024-11-28T06:56:34,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1732776994379, max timestamp=1732776994414Num uniques 1053; 2024-11-28T06:56:34,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=9295, suffix heap size=137480,new segment heap size=21352 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=203 2024-11-28T06:56:34,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1732776994379, max timestamp=1732776994414Num uniques 1053; 2024-11-28T06:56:34,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=7005, suffix heap size=137480,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=153 2024-11-28T06:56:34,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1732776994379, max timestamp=1732776994414Num uniques 1053; 2024-11-28T06:56:34,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=7005, suffix heap size=137480,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=153 2024-11-28T06:56:34,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=262, cellSize=11992, totalHeapSize=27488, min timestamp=1732776994428, max timestamp=1732776994435Num uniques 1012; 2024-11-28T06:56:34,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46342, new segment data size=11992, suffix heap size=133216,new segment heap size=27488 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1012, new segment cells count=262 2024-11-28T06:56:34,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=262, cellSize=11992, totalHeapSize=27488, min timestamp=1732776994428, max timestamp=1732776994435Num uniques 1012; 2024-11-28T06:56:34,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=262, cellSize=11992, totalHeapSize=27488, min timestamp=1732776994428, max timestamp=1732776994435Num uniques 1062; 2024-11-28T06:56:34,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46342, new segment data size=11992, suffix heap size=133216,new segment heap size=27488 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1012, new segment cells count=262 2024-11-28T06:56:34,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48632, new segment data size=11992, suffix heap size=138416,new segment heap size=27488 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1062, new segment cells count=262 2024-11-28T06:56:34,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=221, cellSize=10116, totalHeapSize=23224, min timestamp=1732776994462, max timestamp=1732776994467Num uniques 1121; 2024-11-28T06:56:34,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=221, cellSize=10116, totalHeapSize=23224, min timestamp=1732776994462, max timestamp=1732776994467Num uniques 1121; 2024-11-28T06:56:34,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51336, new segment data size=10116, suffix heap size=144552,new segment heap size=23224 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1121, new segment cells count=221 2024-11-28T06:56:34,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51336, new segment data size=10116, suffix heap size=144552,new segment heap size=23224 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1121, new segment cells count=221 2024-11-28T06:56:34,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=221, cellSize=10116, totalHeapSize=23224, min timestamp=1732776994462, max timestamp=1732776994467Num uniques 1121; 2024-11-28T06:56:34,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51336, new segment data size=10116, suffix heap size=144552,new segment heap size=23224 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1121, new segment cells count=221 2024-11-28T06:56:34,506 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=230, cellSize=10530, totalHeapSize=24160, min timestamp=1732776994494, max timestamp=1732776994503Num uniques 1080; 2024-11-28T06:56:34,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49460, new segment data size=10530, suffix heap size=140288,new segment heap size=24160 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1080, new segment cells count=230 2024-11-28T06:56:34,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=230, cellSize=10530, totalHeapSize=24160, min timestamp=1732776994494, max timestamp=1732776994503Num uniques 1080; 2024-11-28T06:56:34,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49460, new segment data size=10530, suffix heap size=140288,new segment heap size=24160 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1080, new segment cells count=230 2024-11-28T06:56:34,508 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=230, cellSize=10530, totalHeapSize=24160, min timestamp=1732776994494, max timestamp=1732776994503Num uniques 1080; 2024-11-28T06:56:34,508 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49460, new segment data size=10530, suffix heap size=140288,new segment heap size=24160 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1080, new segment cells count=230 2024-11-28T06:56:34,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-28T06:56:34,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:34,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:34,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:34,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:34,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:34,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:34,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=289, cellSize=13234, totalHeapSize=30296, min timestamp=1732776994521, max timestamp=1732776994529Num uniques 1089; 2024-11-28T06:56:34,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=289, cellSize=13234, totalHeapSize=30296, min timestamp=1732776994521, max timestamp=1732776994529Num uniques 1089; 2024-11-28T06:56:34,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49874, new segment data size=13234, suffix heap size=141224,new segment heap size=30296 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1089, new segment cells count=289 2024-11-28T06:56:34,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49874, new segment data size=13234, suffix heap size=141224,new segment heap size=30296 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1089, new segment cells count=289 2024-11-28T06:56:34,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=239, cellSize=10944, totalHeapSize=25096, min timestamp=1732776994521, max timestamp=1732776994530Num uniques 1089; 2024-11-28T06:56:34,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49874, new segment data size=10944, suffix heap size=141224,new segment heap size=25096 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1089, new segment cells count=239 2024-11-28T06:56:34,562 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=248, cellSize=11358, totalHeapSize=26032, min timestamp=1732776994551, max timestamp=1732776994559Num uniques 1148; 2024-11-28T06:56:34,562 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52578, new segment data size=11358, suffix heap size=147360,new segment heap size=26032 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1148, new segment cells count=248 2024-11-28T06:56:34,562 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=298, cellSize=13648, totalHeapSize=31232, min timestamp=1732776994551, max timestamp=1732776994559Num uniques 1148; 2024-11-28T06:56:34,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52578, new segment data size=13648, suffix heap size=147360,new segment heap size=31232 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1148, new segment cells count=298 2024-11-28T06:56:34,566 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=248, cellSize=11358, totalHeapSize=26032, min timestamp=1732776994551, max timestamp=1732776994559Num uniques 1098; 2024-11-28T06:56:34,566 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50288, new segment data size=11358, suffix heap size=142160,new segment heap size=26032 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1098, new segment cells count=248 2024-11-28T06:56:34,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-28T06:56:34,588 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=307, cellSize=14055, totalHeapSize=32168, min timestamp=1732776994576, max timestamp=1732776994585Num uniques 1157; 2024-11-28T06:56:34,588 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=307, cellSize=14055, totalHeapSize=32168, min timestamp=1732776994576, max timestamp=1732776994585Num uniques 1107; 2024-11-28T06:56:34,588 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52985, new segment data size=14055, suffix heap size=148296,new segment heap size=32168 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1157, new segment cells count=307 2024-11-28T06:56:34,588 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50695, new segment data size=14055, suffix heap size=143096,new segment heap size=32168 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1107, new segment cells count=307 2024-11-28T06:56:34,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=307, cellSize=14055, totalHeapSize=32168, min timestamp=1732776994576, max timestamp=1732776994585Num uniques 1107; 2024-11-28T06:56:34,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50695, new segment data size=14055, suffix heap size=143096,new segment heap size=32168 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1107, new segment cells count=307 2024-11-28T06:56:34,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=216, cellSize=9886, totalHeapSize=22704, min timestamp=1732776994605, max timestamp=1732776994607Num uniques 1166; 2024-11-28T06:56:34,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=53396, new segment data size=9886, suffix heap size=149232,new segment heap size=22704 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1166, new segment cells count=216 2024-11-28T06:56:34,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=216, cellSize=9886, totalHeapSize=22704, min timestamp=1732776994605, max timestamp=1732776994607Num uniques 1166; 2024-11-28T06:56:34,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=53396, new segment data size=9886, suffix heap size=149232,new segment heap size=22704 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1166, new segment cells count=216 2024-11-28T06:56:34,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=216, cellSize=9886, totalHeapSize=22704, min timestamp=1732776994605, max timestamp=1732776994607Num uniques 1166; 2024-11-28T06:56:34,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=53396, new segment data size=9886, suffix heap size=149232,new segment heap size=22704 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1166, new segment cells count=216 2024-11-28T06:56:34,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:34,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777054631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:34,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:34,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:34,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:34,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=225, cellSize=10300, totalHeapSize=23640, min timestamp=1732776994627, max timestamp=1732776994634Num uniques 1075; 2024-11-28T06:56:34,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49230, new segment data size=10300, suffix heap size=139768,new segment heap size=23640 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1075, new segment cells count=225 2024-11-28T06:56:34,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=225, cellSize=10293, totalHeapSize=23640, min timestamp=1732776994627, max timestamp=1732776994635Num uniques 1075; 2024-11-28T06:56:34,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=225, cellSize=10300, totalHeapSize=23640, min timestamp=1732776994627, max timestamp=1732776994634Num uniques 1075; 2024-11-28T06:56:34,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49223, new segment data size=10293, suffix heap size=139768,new segment heap size=23640 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1075, new segment cells count=225 2024-11-28T06:56:34,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49230, new segment data size=10300, suffix heap size=139768,new segment heap size=23640 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1075, new segment cells count=225 2024-11-28T06:56:34,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=234, cellSize=10714, totalHeapSize=24576, min timestamp=1732776994651, max timestamp=1732776994658Num uniques 1084; 2024-11-28T06:56:34,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49644, new segment data size=10714, suffix heap size=140704,new segment heap size=24576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1084, new segment cells count=234 2024-11-28T06:56:34,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=234, cellSize=10714, totalHeapSize=24576, min timestamp=1732776994651, max timestamp=1732776994658Num uniques 1084; 2024-11-28T06:56:34,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49644, new segment data size=10714, suffix heap size=140704,new segment heap size=24576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1084, new segment cells count=234 2024-11-28T06:56:34,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=234, cellSize=10714, totalHeapSize=24576, min timestamp=1732776994651, max timestamp=1732776994658Num uniques 1084; 2024-11-28T06:56:34,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49644, new segment data size=10714, suffix heap size=140704,new segment heap size=24576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1084, new segment cells count=234 2024-11-28T06:56:34,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-28T06:56:34,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:34,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:34,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:34,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:34,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:34,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:34,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=293, cellSize=13418, totalHeapSize=30712, min timestamp=1732776994698, max timestamp=1732776994713Num uniques 1093; 2024-11-28T06:56:34,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50058, new segment data size=13418, suffix heap size=141640,new segment heap size=30712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1093, new segment cells count=293 2024-11-28T06:56:34,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=193, cellSize=8838, totalHeapSize=20312, min timestamp=1732776994706, max timestamp=1732776994713Num uniques 1093; 2024-11-28T06:56:34,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50058, new segment data size=8838, suffix heap size=141640,new segment heap size=20312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1093, new segment cells count=193 2024-11-28T06:56:34,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=293, cellSize=13418, totalHeapSize=30712, min timestamp=1732776994698, max timestamp=1732776994713Num uniques 1093; 2024-11-28T06:56:34,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50058, new segment data size=13418, suffix heap size=141640,new segment heap size=30712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1093, new segment cells count=293 2024-11-28T06:56:34,731 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/2419ee7c3e424462bfeaf81814ddadf5 2024-11-28T06:56:34,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/d997a356a4704564b281616ba3967bbc is 50, key is test_row_0/C:col10/1732776992140/Put/seqid=0 2024-11-28T06:56:34,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742294_1470 (size=12001) 2024-11-28T06:56:34,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=252, cellSize=11540, totalHeapSize=26448, min timestamp=1732776994734, max timestamp=1732776994744Num uniques 1052; 2024-11-28T06:56:34,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48180, new segment data size=11540, suffix heap size=137376,new segment heap size=26448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1052, new segment cells count=252 2024-11-28T06:56:34,748 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=252, cellSize=11540, totalHeapSize=26448, min timestamp=1732776994734, max timestamp=1732776994744Num uniques 1152; 2024-11-28T06:56:34,748 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52760, new segment data size=11540, suffix heap size=147776,new segment heap size=26448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1152, new segment cells count=252 2024-11-28T06:56:34,748 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=252, cellSize=11540, totalHeapSize=26448, min timestamp=1732776994734, max timestamp=1732776994744Num uniques 1152; 2024-11-28T06:56:34,748 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52760, new segment data size=11540, suffix heap size=147776,new segment heap size=26448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1152, new segment cells count=252 2024-11-28T06:56:34,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:34,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=261, cellSize=11946, totalHeapSize=27384, min timestamp=1732776994760, max timestamp=1732776994766Num uniques 1111; 2024-11-28T06:56:34,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50876, new segment data size=11946, suffix heap size=143512,new segment heap size=27384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1111, new segment cells count=261 2024-11-28T06:56:34,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777054765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:34,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=261, cellSize=11946, totalHeapSize=27384, min timestamp=1732776994760, max timestamp=1732776994766Num uniques 1111; 2024-11-28T06:56:34,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=261, cellSize=11946, totalHeapSize=27384, min timestamp=1732776994760, max timestamp=1732776994766Num uniques 1111; 2024-11-28T06:56:34,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50876, new segment data size=11946, suffix heap size=143512,new segment heap size=27384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1111, new segment cells count=261 2024-11-28T06:56:34,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50876, new segment data size=11946, suffix heap size=143512,new segment heap size=27384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1111, new segment cells count=261 2024-11-28T06:56:34,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:34,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:34,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:34,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:34,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777054792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:34,797 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:34,797 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:34,797 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:34,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=220, cellSize=10070, totalHeapSize=23120, min timestamp=1732776994777, max timestamp=1732776994794Num uniques 1120; 2024-11-28T06:56:34,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51290, new segment data size=10070, suffix heap size=144448,new segment heap size=23120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1120, new segment cells count=220 2024-11-28T06:56:34,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=220, cellSize=10070, totalHeapSize=23120, min timestamp=1732776994777, max timestamp=1732776994794Num uniques 1120; 2024-11-28T06:56:34,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=220, cellSize=10070, totalHeapSize=23120, min timestamp=1732776994777, max timestamp=1732776994794Num uniques 1120; 2024-11-28T06:56:34,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51290, new segment data size=10070, suffix heap size=144448,new segment heap size=23120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1120, new segment cells count=220 2024-11-28T06:56:34,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51290, new segment data size=10070, suffix heap size=144448,new segment heap size=23120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1120, new segment cells count=220 2024-11-28T06:56:34,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=279, cellSize=12774, totalHeapSize=29256, min timestamp=1732776994813, max timestamp=1732776994822Num uniques 1079; 2024-11-28T06:56:34,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=279, cellSize=12774, totalHeapSize=29256, min timestamp=1732776994813, max timestamp=1732776994822Num uniques 1079; 2024-11-28T06:56:34,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=279, cellSize=12774, totalHeapSize=29256, min timestamp=1732776994813, max timestamp=1732776994822Num uniques 1079; 2024-11-28T06:56:34,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49414, new segment data size=12774, suffix heap size=140184,new segment heap size=29256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1079, new segment cells count=279 2024-11-28T06:56:34,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49414, new segment data size=12774, suffix heap size=140184,new segment heap size=29256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1079, new segment cells count=279 2024-11-28T06:56:34,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49414, new segment data size=12774, suffix heap size=140184,new segment heap size=29256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1079, new segment cells count=279 2024-11-28T06:56:34,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-28T06:56:34,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:34,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:34,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:34,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:34,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:34,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:34,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:34,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42204 deadline: 1732777054853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:34,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:34,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:34,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:34,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=188, cellSize=8608, totalHeapSize=19792, min timestamp=1732776994839, max timestamp=1732776994856Num uniques 1138; 2024-11-28T06:56:34,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52118, new segment data size=8608, suffix heap size=146320,new segment heap size=19792 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1138, new segment cells count=188 2024-11-28T06:56:34,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=188, cellSize=8607, totalHeapSize=19792, min timestamp=1732776994839, max timestamp=1732776994856Num uniques 1138; 2024-11-28T06:56:34,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=188, cellSize=8598, totalHeapSize=19792, min timestamp=1732776994839, max timestamp=1732776994856Num uniques 1138; 2024-11-28T06:56:34,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52117, new segment data size=8607, suffix heap size=146320,new segment heap size=19792 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1138, new segment cells count=188 2024-11-28T06:56:34,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52108, new segment data size=8598, suffix heap size=146320,new segment heap size=19792 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1138, new segment cells count=188 2024-11-28T06:56:34,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=197, cellSize=9022, totalHeapSize=20728, min timestamp=1732776994880, max timestamp=1732776994888Num uniques 1047; 2024-11-28T06:56:34,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47952, new segment data size=9022, suffix heap size=136856,new segment heap size=20728 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1047, new segment cells count=197 2024-11-28T06:56:34,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=197, cellSize=9022, totalHeapSize=20728, min timestamp=1732776994880, max timestamp=1732776994888Num uniques 1047; 2024-11-28T06:56:34,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47952, new segment data size=9022, suffix heap size=136856,new segment heap size=20728 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1047, new segment cells count=197 2024-11-28T06:56:34,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=197, cellSize=9022, totalHeapSize=20728, min timestamp=1732776994880, max timestamp=1732776994888Num uniques 1047; 2024-11-28T06:56:34,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47952, new segment data size=9022, suffix heap size=136856,new segment heap size=20728 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1047, new segment cells count=197 2024-11-28T06:56:34,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=256, cellSize=11720, totalHeapSize=26864, min timestamp=1732776994908, max timestamp=1732776994918Num uniques 1056; 2024-11-28T06:56:34,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=256, cellSize=11720, totalHeapSize=26864, min timestamp=1732776994908, max timestamp=1732776994918Num uniques 1056; 2024-11-28T06:56:34,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48360, new segment data size=11720, suffix heap size=137792,new segment heap size=26864 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1056, new segment cells count=256 2024-11-28T06:56:34,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48360, new segment data size=11720, suffix heap size=137792,new segment heap size=26864 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1056, new segment cells count=256 2024-11-28T06:56:34,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=256, cellSize=11720, totalHeapSize=26864, min timestamp=1732776994908, max timestamp=1732776994918Num uniques 1056; 2024-11-28T06:56:34,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48360, new segment data size=11720, suffix heap size=137792,new segment heap size=26864 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1056, new segment cells count=256 2024-11-28T06:56:34,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=315, cellSize=14420, totalHeapSize=33000, min timestamp=1732776994936, max timestamp=1732776994945Num uniques 1115; 2024-11-28T06:56:34,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51060, new segment data size=14420, suffix heap size=143928,new segment heap size=33000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1115, new segment cells count=315 2024-11-28T06:56:34,949 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=315, cellSize=14420, totalHeapSize=33000, min timestamp=1732776994936, max timestamp=1732776994945Num uniques 1115; 2024-11-28T06:56:34,949 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51060, new segment data size=14420, suffix heap size=143928,new segment heap size=33000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1115, new segment cells count=315 2024-11-28T06:56:34,949 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=315, cellSize=14420, totalHeapSize=33000, min timestamp=1732776994936, max timestamp=1732776994945Num uniques 1115; 2024-11-28T06:56:34,949 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51060, new segment data size=14420, suffix heap size=143928,new segment heap size=33000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1115, new segment cells count=315 2024-11-28T06:56:34,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-28T06:56:34,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:34,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:34,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:34,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:34,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:34,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:35,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:35,015 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=174, cellSize=7958, totalHeapSize=18336, min timestamp=1732776995000, max timestamp=1732776995012Num uniques 1174; 2024-11-28T06:56:35,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42204 deadline: 1732777055004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:35,015 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=53758, new segment data size=7958, suffix heap size=150064,new segment heap size=18336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1174, new segment cells count=174 2024-11-28T06:56:35,015 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=174, cellSize=7964, totalHeapSize=18336, min timestamp=1732776995000, max timestamp=1732776995012Num uniques 1174; 2024-11-28T06:56:35,015 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=174, cellSize=7964, totalHeapSize=18336, min timestamp=1732776995000, max timestamp=1732776995011Num uniques 1174; 2024-11-28T06:56:35,015 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=53764, new segment data size=7964, suffix heap size=150064,new segment heap size=18336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1174, new segment cells count=174 2024-11-28T06:56:35,015 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=53764, new segment data size=7964, suffix heap size=150064,new segment heap size=18336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1174, new segment cells count=174 2024-11-28T06:56:35,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:35,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=233, cellSize=10668, totalHeapSize=24472, min timestamp=1732776995034, max timestamp=1732776995044Num uniques 1033; 2024-11-28T06:56:35,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=183, cellSize=8378, totalHeapSize=19272, min timestamp=1732776995034, max timestamp=1732776995044Num uniques 1033; 2024-11-28T06:56:35,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47308, new segment data size=10668, suffix heap size=135400,new segment heap size=24472 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1033, new segment cells count=233 2024-11-28T06:56:35,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47308, new segment data size=8378, suffix heap size=135400,new segment heap size=19272 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1033, new segment cells count=183 2024-11-28T06:56:35,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=183, cellSize=8378, totalHeapSize=19272, min timestamp=1732776995034, max timestamp=1732776995044Num uniques 1033; 2024-11-28T06:56:35,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47308, new segment data size=8378, suffix heap size=135400,new segment heap size=19272 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1033, new segment cells count=183 2024-11-28T06:56:35,099 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=192, cellSize=8792, totalHeapSize=20208, min timestamp=1732776995064, max timestamp=1732776995095Num uniques 1092; 2024-11-28T06:56:35,099 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=192, cellSize=8792, totalHeapSize=20208, min timestamp=1732776995064, max timestamp=1732776995095Num uniques 1042; 2024-11-28T06:56:35,099 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50012, new segment data size=8792, suffix heap size=141536,new segment heap size=20208 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1092, new segment cells count=192 2024-11-28T06:56:35,099 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47722, new segment data size=8792, suffix heap size=136336,new segment heap size=20208 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1042, new segment cells count=192 2024-11-28T06:56:35,099 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=192, cellSize=8792, totalHeapSize=20208, min timestamp=1732776995064, max timestamp=1732776995095Num uniques 1042; 2024-11-28T06:56:35,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47722, new segment data size=8792, suffix heap size=136336,new segment heap size=20208 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1042, new segment cells count=192 2024-11-28T06:56:35,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:35,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42204 deadline: 1732777055126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:35,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:35,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=201, cellSize=9205, totalHeapSize=21144, min timestamp=1732776995119, max timestamp=1732776995126Num uniques 1051; 2024-11-28T06:56:35,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48135, new segment data size=9205, suffix heap size=137272,new segment heap size=21144 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1051, new segment cells count=201 2024-11-28T06:56:35,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=201, cellSize=9205, totalHeapSize=21144, min timestamp=1732776995119, max timestamp=1732776995126Num uniques 1051; 2024-11-28T06:56:35,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48135, new segment data size=9205, suffix heap size=137272,new segment heap size=21144 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1051, new segment cells count=201 2024-11-28T06:56:35,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=201, cellSize=9205, totalHeapSize=21144, min timestamp=1732776995119, max timestamp=1732776995126Num uniques 1051; 2024-11-28T06:56:35,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48135, new segment data size=9205, suffix heap size=137272,new segment heap size=21144 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1051, new segment cells count=201 2024-11-28T06:56:35,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-28T06:56:35,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:35,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:35,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:35,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:35,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:35,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:35,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/d997a356a4704564b281616ba3967bbc 2024-11-28T06:56:35,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/899e47f9a9c14af5b1860932d68af59e as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/899e47f9a9c14af5b1860932d68af59e 2024-11-28T06:56:35,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/899e47f9a9c14af5b1860932d68af59e, entries=150, sequenceid=70, filesize=30.2 K 2024-11-28T06:56:35,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/2419ee7c3e424462bfeaf81814ddadf5 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/2419ee7c3e424462bfeaf81814ddadf5 2024-11-28T06:56:35,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/2419ee7c3e424462bfeaf81814ddadf5, entries=150, sequenceid=70, filesize=11.7 K 2024-11-28T06:56:35,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/d997a356a4704564b281616ba3967bbc as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/d997a356a4704564b281616ba3967bbc 2024-11-28T06:56:35,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/d997a356a4704564b281616ba3967bbc, entries=150, sequenceid=70, filesize=11.7 K 2024-11-28T06:56:35,155 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=140.89 KB/144270 for d1f05a2e2429924e6c4e8d32d50aa9ca in 1654ms, sequenceid=70, compaction requested=false 2024-11-28T06:56:35,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:35,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=260, cellSize=11900, totalHeapSize=27280, min timestamp=1732776995144, max timestamp=1732776995154Num uniques 1060; 2024-11-28T06:56:35,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48540, new segment data size=11900, suffix heap size=138208,new segment heap size=27280 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1060, new segment cells count=260 2024-11-28T06:56:35,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=260, cellSize=11900, totalHeapSize=27280, min timestamp=1732776995144, max timestamp=1732776995154Num uniques 1060; 2024-11-28T06:56:35,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48540, new segment data size=11900, suffix heap size=138208,new segment heap size=27280 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1060, new segment cells count=260 2024-11-28T06:56:35,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=260, cellSize=11900, totalHeapSize=27280, min timestamp=1732776995144, max timestamp=1732776995154Num uniques 1060; 2024-11-28T06:56:35,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48540, new segment data size=11900, suffix heap size=138208,new segment heap size=27280 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1060, new segment cells count=260 2024-11-28T06:56:35,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:35,162 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1f05a2e2429924e6c4e8d32d50aa9ca 3/3 column families, dataSize=53.67 KB heapSize=136.45 KB 2024-11-28T06:56:35,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=A 2024-11-28T06:56:35,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:56:35,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=B 2024-11-28T06:56:35,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:56:35,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=C 2024-11-28T06:56:35,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:56:35,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411284c9fc08cf437472ea265ec02d4b9718b_d1f05a2e2429924e6c4e8d32d50aa9ca is 50, key is test_row_0/A:col10/1732776995161/Put/seqid=0 2024-11-28T06:56:35,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742295_1471 (size=14994) 2024-11-28T06:56:35,175 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:35,181 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411284c9fc08cf437472ea265ec02d4b9718b_d1f05a2e2429924e6c4e8d32d50aa9ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411284c9fc08cf437472ea265ec02d4b9718b_d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:35,182 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/df07587d9d3f4656ac2412fbd7bfc225, store: [table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:35,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/df07587d9d3f4656ac2412fbd7bfc225 is 175, key is test_row_0/A:col10/1732776995161/Put/seqid=0 2024-11-28T06:56:35,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742296_1472 (size=39949) 2024-11-28T06:56:35,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:35,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42208 deadline: 1732777055191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:35,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42230 deadline: 1732777055192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:35,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732776995188, max timestamp=1732776995195Num uniques 859; 2024-11-28T06:56:35,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732776995188, max timestamp=1732776995195Num uniques 859; 2024-11-28T06:56:35,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732776995188, max timestamp=1732776995195Num uniques 859; 2024-11-28T06:56:35,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T06:56:35,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T06:56:35,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1cb9cf437a36:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T06:56:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:35,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:35,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777055245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:35,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:35,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:35,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777055249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:35,254 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,254 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,254 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:35,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-28T06:56:35,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:35,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:35,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:35,287 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:35,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:35,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:35,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:35,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42208 deadline: 1732777055306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:35,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:35,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42230 deadline: 1732777055306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:35,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:35,311 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,311 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,311 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:35,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:35,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42204 deadline: 1732777055335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:35,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:35,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777055355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:35,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:35,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:35,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777055366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:35,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:35,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-28T06:56:35,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:35,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:35,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:35,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:35,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:35,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:35,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:35,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42208 deadline: 1732777055515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:35,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:35,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:35,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42230 deadline: 1732777055515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:35,520 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,520 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,520 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:35,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:35,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777055565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:35,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:35,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:35,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777055576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:35,577 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,577 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,577 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:35,588 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=767, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/df07587d9d3f4656ac2412fbd7bfc225 2024-11-28T06:56:35,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-28T06:56:35,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:35,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:35,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:35,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:35,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:35,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:35,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/5403d6220f514f949b1f7e51e2b271e8 is 50, key is test_row_0/B:col10/1732776995161/Put/seqid=0 2024-11-28T06:56:35,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742297_1473 (size=12301) 2024-11-28T06:56:35,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-28T06:56:35,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:35,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42204 deadline: 1732777055645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:35,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,650 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:35,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-28T06:56:35,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:35,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:35,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:35,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:35,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:35,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:35,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42208 deadline: 1732777055826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:35,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42230 deadline: 1732777055826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:35,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:35,829 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,829 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,829 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:35,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777055875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:35,881 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,881 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,881 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:35,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777055886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:35,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:35,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:35,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-28T06:56:35,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:35,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:35,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:35,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:35,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:35,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=767 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/5403d6220f514f949b1f7e51e2b271e8 2024-11-28T06:56:36,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/40575c113faf4614a3f5c9fe36d4cd5a is 50, key is test_row_0/C:col10/1732776995161/Put/seqid=0 2024-11-28T06:56:36,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742298_1474 (size=12301) 2024-11-28T06:56:36,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-28T06:56:36,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:36,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:36,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:36,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:36,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:36,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:36,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:36,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42204 deadline: 1732777056156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:36,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:36,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:36,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:36,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-28T06:56:36,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:36,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:36,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:36,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:36,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:36,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:36,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:36,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42208 deadline: 1732777056335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:36,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:36,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:36,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:36,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:36,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42230 deadline: 1732777056335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:36,339 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:36,339 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:36,339 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:36,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-28T06:56:36,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:36,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:36,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:36,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:36,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:36,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:36,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:36,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777056386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:36,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:36,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:36,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:36,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:36,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777056395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:36,399 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:36,399 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:36,399 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:36,406 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=767 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/40575c113faf4614a3f5c9fe36d4cd5a 2024-11-28T06:56:36,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/df07587d9d3f4656ac2412fbd7bfc225 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/df07587d9d3f4656ac2412fbd7bfc225 2024-11-28T06:56:36,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/df07587d9d3f4656ac2412fbd7bfc225, entries=200, sequenceid=767, filesize=39.0 K 2024-11-28T06:56:36,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/5403d6220f514f949b1f7e51e2b271e8 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/5403d6220f514f949b1f7e51e2b271e8 2024-11-28T06:56:36,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/5403d6220f514f949b1f7e51e2b271e8, entries=150, sequenceid=767, filesize=12.0 K 2024-11-28T06:56:36,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/40575c113faf4614a3f5c9fe36d4cd5a as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/40575c113faf4614a3f5c9fe36d4cd5a 2024-11-28T06:56:36,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/40575c113faf4614a3f5c9fe36d4cd5a, entries=150, sequenceid=767, filesize=12.0 K 2024-11-28T06:56:36,419 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~156.33 KB/160080, currentSize=127.47 KB/130530 for d1f05a2e2429924e6c4e8d32d50aa9ca in 1257ms, sequenceid=767, compaction requested=true 2024-11-28T06:56:36,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:36,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f05a2e2429924e6c4e8d32d50aa9ca:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:56:36,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:36,420 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:56:36,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f05a2e2429924e6c4e8d32d50aa9ca:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:56:36,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:36,420 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:56:36,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f05a2e2429924e6c4e8d32d50aa9ca:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:56:36,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:56:36,420 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101962 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:56:36,420 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36406 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:56:36,420 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): d1f05a2e2429924e6c4e8d32d50aa9ca/A is initiating minor compaction (all files) 2024-11-28T06:56:36,420 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): d1f05a2e2429924e6c4e8d32d50aa9ca/B is initiating minor compaction (all files) 2024-11-28T06:56:36,420 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f05a2e2429924e6c4e8d32d50aa9ca/A in TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:36,420 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f05a2e2429924e6c4e8d32d50aa9ca/B in TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:36,421 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/f89b89d6c42a433a8a74ee42ca16b94e, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/2419ee7c3e424462bfeaf81814ddadf5, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/5403d6220f514f949b1f7e51e2b271e8] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp, totalSize=35.6 K 2024-11-28T06:56:36,421 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/6838b68679db4f309cd6da111982fe0d, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/899e47f9a9c14af5b1860932d68af59e, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/df07587d9d3f4656ac2412fbd7bfc225] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp, totalSize=99.6 K 2024-11-28T06:56:36,421 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:36,421 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. files: [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/6838b68679db4f309cd6da111982fe0d, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/899e47f9a9c14af5b1860932d68af59e, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/df07587d9d3f4656ac2412fbd7bfc225] 2024-11-28T06:56:36,421 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6838b68679db4f309cd6da111982fe0d, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732776991466 2024-11-28T06:56:36,421 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting f89b89d6c42a433a8a74ee42ca16b94e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732776991466 2024-11-28T06:56:36,421 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 2419ee7c3e424462bfeaf81814ddadf5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1732776992140 2024-11-28T06:56:36,421 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 899e47f9a9c14af5b1860932d68af59e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1732776992140 2024-11-28T06:56:36,421 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 5403d6220f514f949b1f7e51e2b271e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=767, earliestPutTs=1732776995160 2024-11-28T06:56:36,421 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting df07587d9d3f4656ac2412fbd7bfc225, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=767, earliestPutTs=1732776995154 2024-11-28T06:56:36,426 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:36,428 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f05a2e2429924e6c4e8d32d50aa9ca#B#compaction#401 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:36,429 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/d20de09ef09b460492c9eade728aa14d is 50, key is test_row_0/B:col10/1732776995161/Put/seqid=0 2024-11-28T06:56:36,437 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128c4f116b7adb349b0b5eed6c7d076c8e2_d1f05a2e2429924e6c4e8d32d50aa9ca store=[table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:36,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742299_1475 (size=12507) 2024-11-28T06:56:36,440 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128c4f116b7adb349b0b5eed6c7d076c8e2_d1f05a2e2429924e6c4e8d32d50aa9ca, store=[table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:36,440 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128c4f116b7adb349b0b5eed6c7d076c8e2_d1f05a2e2429924e6c4e8d32d50aa9ca because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:36,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742300_1476 (size=4469) 2024-11-28T06:56:36,451 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f05a2e2429924e6c4e8d32d50aa9ca#A#compaction#400 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:36,451 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/e7388049d2ca4e8b89848a278677bede is 175, key is test_row_0/A:col10/1732776995161/Put/seqid=0 2024-11-28T06:56:36,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742301_1477 (size=31461) 2024-11-28T06:56:36,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-28T06:56:36,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:36,506 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2902): Flushing d1f05a2e2429924e6c4e8d32d50aa9ca 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-28T06:56:36,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=A 2024-11-28T06:56:36,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:56:36,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=B 2024-11-28T06:56:36,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:56:36,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=C 2024-11-28T06:56:36,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:56:36,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128d63e32d2a889415692feedcc04db1718_d1f05a2e2429924e6c4e8d32d50aa9ca is 50, key is test_row_0/A:col10/1732776995225/Put/seqid=0 2024-11-28T06:56:36,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742302_1478 (size=12454) 2024-11-28T06:56:36,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:36,519 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128d63e32d2a889415692feedcc04db1718_d1f05a2e2429924e6c4e8d32d50aa9ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d63e32d2a889415692feedcc04db1718_d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:36,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/ca575823ea24400593133a4af98fd990, store: [table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:36,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/ca575823ea24400593133a4af98fd990 is 175, key is test_row_0/A:col10/1732776995225/Put/seqid=0 2024-11-28T06:56:36,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742303_1479 (size=31255) 2024-11-28T06:56:36,843 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/d20de09ef09b460492c9eade728aa14d as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/d20de09ef09b460492c9eade728aa14d 2024-11-28T06:56:36,848 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f05a2e2429924e6c4e8d32d50aa9ca/B of d1f05a2e2429924e6c4e8d32d50aa9ca into d20de09ef09b460492c9eade728aa14d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:36,848 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:36,848 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., storeName=d1f05a2e2429924e6c4e8d32d50aa9ca/B, priority=13, startTime=1732776996420; duration=0sec 2024-11-28T06:56:36,848 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:56:36,848 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f05a2e2429924e6c4e8d32d50aa9ca:B 2024-11-28T06:56:36,848 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:56:36,849 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36406 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:56:36,849 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): d1f05a2e2429924e6c4e8d32d50aa9ca/C is initiating minor compaction (all files) 2024-11-28T06:56:36,849 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f05a2e2429924e6c4e8d32d50aa9ca/C in TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:36,849 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/8d213d83924943e1b3b5d15fe352d67a, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/d997a356a4704564b281616ba3967bbc, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/40575c113faf4614a3f5c9fe36d4cd5a] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp, totalSize=35.6 K 2024-11-28T06:56:36,849 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d213d83924943e1b3b5d15fe352d67a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732776991466 2024-11-28T06:56:36,850 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting d997a356a4704564b281616ba3967bbc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1732776992140 2024-11-28T06:56:36,850 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 40575c113faf4614a3f5c9fe36d4cd5a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=767, earliestPutTs=1732776995160 2024-11-28T06:56:36,855 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f05a2e2429924e6c4e8d32d50aa9ca#C#compaction#403 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:36,855 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/56a61576bc4b4e939d4381491640816c is 50, key is test_row_0/C:col10/1732776995161/Put/seqid=0 2024-11-28T06:56:36,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742304_1480 (size=12507) 2024-11-28T06:56:36,858 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/e7388049d2ca4e8b89848a278677bede as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/e7388049d2ca4e8b89848a278677bede 2024-11-28T06:56:36,862 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/56a61576bc4b4e939d4381491640816c as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/56a61576bc4b4e939d4381491640816c 2024-11-28T06:56:36,863 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f05a2e2429924e6c4e8d32d50aa9ca/A of d1f05a2e2429924e6c4e8d32d50aa9ca into e7388049d2ca4e8b89848a278677bede(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:36,863 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:36,863 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., storeName=d1f05a2e2429924e6c4e8d32d50aa9ca/A, priority=13, startTime=1732776996419; duration=0sec 2024-11-28T06:56:36,863 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:36,863 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f05a2e2429924e6c4e8d32d50aa9ca:A 2024-11-28T06:56:36,866 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f05a2e2429924e6c4e8d32d50aa9ca/C of d1f05a2e2429924e6c4e8d32d50aa9ca into 56a61576bc4b4e939d4381491640816c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:36,866 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:36,866 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., storeName=d1f05a2e2429924e6c4e8d32d50aa9ca/C, priority=13, startTime=1732776996420; duration=0sec 2024-11-28T06:56:36,866 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:36,866 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f05a2e2429924e6c4e8d32d50aa9ca:C 2024-11-28T06:56:36,924 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=802, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/ca575823ea24400593133a4af98fd990 2024-11-28T06:56:36,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/369cbc5925484469bb32ef6df9ed592f is 50, key is test_row_0/B:col10/1732776995225/Put/seqid=0 2024-11-28T06:56:36,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742305_1481 (size=12301) 2024-11-28T06:56:37,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:37,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:37,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:37,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42204 deadline: 1732777057206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:37,211 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:37,211 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:37,211 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:37,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42204 deadline: 1732777057316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:37,319 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:37,319 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:37,319 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:37,333 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=802 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/369cbc5925484469bb32ef6df9ed592f 2024-11-28T06:56:37,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/0cbb43648fac46cfa57c06a5f70c393c is 50, key is test_row_0/C:col10/1732776995225/Put/seqid=0 2024-11-28T06:56:37,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742306_1482 (size=12301) 2024-11-28T06:56:37,342 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=802 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/0cbb43648fac46cfa57c06a5f70c393c 2024-11-28T06:56:37,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/ca575823ea24400593133a4af98fd990 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/ca575823ea24400593133a4af98fd990 2024-11-28T06:56:37,352 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/ca575823ea24400593133a4af98fd990, entries=150, sequenceid=802, filesize=30.5 K 2024-11-28T06:56:37,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/369cbc5925484469bb32ef6df9ed592f as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/369cbc5925484469bb32ef6df9ed592f 2024-11-28T06:56:37,355 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/369cbc5925484469bb32ef6df9ed592f, entries=150, sequenceid=802, filesize=12.0 K 2024-11-28T06:56:37,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/0cbb43648fac46cfa57c06a5f70c393c as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/0cbb43648fac46cfa57c06a5f70c393c 2024-11-28T06:56:37,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:37,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42208 deadline: 1732777057355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:37,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:37,359 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_1' in d1f05a2e2429924e6c4e8d32d50aa9ca of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:56:37,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42230 deadline: 1732777057355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:37,359 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/0cbb43648fac46cfa57c06a5f70c393c, entries=150, sequenceid=802, filesize=12.0 K 2024-11-28T06:56:37,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:37,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:37,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:37,360 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_1' in d1f05a2e2429924e6c4e8d32d50aa9ca of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:56:37,360 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for d1f05a2e2429924e6c4e8d32d50aa9ca in 854ms, sequenceid=802, compaction requested=false 2024-11-28T06:56:37,360 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:37,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2603): Flush status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:37,360 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:37,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:37,360 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:37,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=143 2024-11-28T06:56:37,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=143 2024-11-28T06:56:37,363 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-28T06:56:37,363 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.9130 sec 2024-11-28T06:56:37,365 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 3.9180 sec 2024-11-28T06:56:37,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:37,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1f05a2e2429924e6c4e8d32d50aa9ca 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-28T06:56:37,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=A 2024-11-28T06:56:37,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:37,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=B 2024-11-28T06:56:37,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:37,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=C 2024-11-28T06:56:37,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:37,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112801bedd90a52b46c997626c9b2b30dab4_d1f05a2e2429924e6c4e8d32d50aa9ca is 50, key is test_row_0/A:col10/1732776997407/Put/seqid=0 2024-11-28T06:56:37,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742307_1483 (size=14994) 2024-11-28T06:56:37,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:37,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777057462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:37,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:37,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:37,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:37,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:37,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777057464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:37,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:37,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:37,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:37,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:37,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42204 deadline: 1732777057526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:37,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:37,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:37,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:37,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:37,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777057575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:37,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:37,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:37,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:37,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:37,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777057575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:37,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:37,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:37,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:37,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-28T06:56:37,616 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:56:37,617 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:56:37,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-28T06:56:37,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-28T06:56:37,618 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:56:37,619 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:56:37,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:56:37,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-28T06:56:37,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-28T06:56:37,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:37,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:37,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:37,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:37,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:37,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:37,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:37,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777057785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:37,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:37,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:37,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:37,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:37,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777057785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:37,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:37,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:37,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:37,819 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:37,822 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112801bedd90a52b46c997626c9b2b30dab4_d1f05a2e2429924e6c4e8d32d50aa9ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112801bedd90a52b46c997626c9b2b30dab4_d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:37,822 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/adb7b57d8efd4147913baf1dfc8f22c9, store: [table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:37,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/adb7b57d8efd4147913baf1dfc8f22c9 is 175, key is test_row_0/A:col10/1732776997407/Put/seqid=0 2024-11-28T06:56:37,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742308_1484 (size=39949) 2024-11-28T06:56:37,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:37,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42204 deadline: 1732777057835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:37,838 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:37,838 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:37,838 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:37,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-28T06:56:37,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:37,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:37,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:37,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:37,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:37,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:37,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-28T06:56:38,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-28T06:56:38,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:38,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:38,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:38,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:38,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:38,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:38,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:38,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777058096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:38,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:38,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777058096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:38,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:38,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:38,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:38,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:38,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:38,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:38,230 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=817, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/adb7b57d8efd4147913baf1dfc8f22c9 2024-11-28T06:56:38,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-28T06:56:38,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:38,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:38,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:38,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:38,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:38,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:38,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/0d329cd372204c2dbef84b23d3313bb1 is 50, key is test_row_0/B:col10/1732776997407/Put/seqid=0 2024-11-28T06:56:38,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-28T06:56:38,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742309_1485 (size=12301) 2024-11-28T06:56:38,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:38,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42204 deadline: 1732777058345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:38,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:38,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:38,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:38,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-28T06:56:38,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:38,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:38,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:38,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:38,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:38,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:38,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-28T06:56:38,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:38,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:38,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:38,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:38,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:38,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:38,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:38,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777058615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:38,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:38,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:38,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:38,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:38,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777058615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:38,618 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:38,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:38,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:38,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=817 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/0d329cd372204c2dbef84b23d3313bb1 2024-11-28T06:56:38,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/139242ce9eb8452c8744cc6c47ae23c5 is 50, key is test_row_0/C:col10/1732776997407/Put/seqid=0 2024-11-28T06:56:38,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-28T06:56:38,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:38,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:38,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:38,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:38,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:38,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:38,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742310_1486 (size=12301) 2024-11-28T06:56:38,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=817 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/139242ce9eb8452c8744cc6c47ae23c5 2024-11-28T06:56:38,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/adb7b57d8efd4147913baf1dfc8f22c9 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/adb7b57d8efd4147913baf1dfc8f22c9 2024-11-28T06:56:38,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/adb7b57d8efd4147913baf1dfc8f22c9, entries=200, sequenceid=817, filesize=39.0 K 2024-11-28T06:56:38,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/0d329cd372204c2dbef84b23d3313bb1 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/0d329cd372204c2dbef84b23d3313bb1 2024-11-28T06:56:38,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/0d329cd372204c2dbef84b23d3313bb1, entries=150, sequenceid=817, filesize=12.0 K 2024-11-28T06:56:38,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/139242ce9eb8452c8744cc6c47ae23c5 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/139242ce9eb8452c8744cc6c47ae23c5 2024-11-28T06:56:38,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/139242ce9eb8452c8744cc6c47ae23c5, entries=150, sequenceid=817, filesize=12.0 K 2024-11-28T06:56:38,733 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for d1f05a2e2429924e6c4e8d32d50aa9ca in 1325ms, sequenceid=817, compaction requested=true 2024-11-28T06:56:38,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:38,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f05a2e2429924e6c4e8d32d50aa9ca:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:56:38,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:38,734 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:56:38,734 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:56:38,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f05a2e2429924e6c4e8d32d50aa9ca:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:56:38,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:38,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f05a2e2429924e6c4e8d32d50aa9ca:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:56:38,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:56:38,735 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37109 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:56:38,735 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): d1f05a2e2429924e6c4e8d32d50aa9ca/B is initiating minor compaction (all files) 2024-11-28T06:56:38,735 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f05a2e2429924e6c4e8d32d50aa9ca/B in TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:38,735 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/d20de09ef09b460492c9eade728aa14d, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/369cbc5925484469bb32ef6df9ed592f, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/0d329cd372204c2dbef84b23d3313bb1] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp, totalSize=36.2 K 2024-11-28T06:56:38,735 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102665 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:56:38,735 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): d1f05a2e2429924e6c4e8d32d50aa9ca/A is initiating minor compaction (all files) 2024-11-28T06:56:38,735 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f05a2e2429924e6c4e8d32d50aa9ca/A in TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:38,735 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/e7388049d2ca4e8b89848a278677bede, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/ca575823ea24400593133a4af98fd990, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/adb7b57d8efd4147913baf1dfc8f22c9] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp, totalSize=100.3 K 2024-11-28T06:56:38,735 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:38,735 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. files: [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/e7388049d2ca4e8b89848a278677bede, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/ca575823ea24400593133a4af98fd990, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/adb7b57d8efd4147913baf1dfc8f22c9] 2024-11-28T06:56:38,736 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting d20de09ef09b460492c9eade728aa14d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=767, earliestPutTs=1732776995160 2024-11-28T06:56:38,736 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7388049d2ca4e8b89848a278677bede, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=767, earliestPutTs=1732776995160 2024-11-28T06:56:38,736 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 369cbc5925484469bb32ef6df9ed592f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=802, earliestPutTs=1732776995225 2024-11-28T06:56:38,736 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca575823ea24400593133a4af98fd990, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=802, earliestPutTs=1732776995225 2024-11-28T06:56:38,736 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d329cd372204c2dbef84b23d3313bb1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=817, earliestPutTs=1732776997195 2024-11-28T06:56:38,738 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting adb7b57d8efd4147913baf1dfc8f22c9, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=817, earliestPutTs=1732776997188 2024-11-28T06:56:38,748 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f05a2e2429924e6c4e8d32d50aa9ca#B#compaction#409 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:38,749 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/2eda224d6af34beabe1211f9e212f9b9 is 50, key is test_row_0/B:col10/1732776997407/Put/seqid=0 2024-11-28T06:56:38,752 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:38,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-28T06:56:38,771 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112802b21ca383cc4ae4b02858eb03b14456_d1f05a2e2429924e6c4e8d32d50aa9ca store=[table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:38,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742311_1487 (size=12609) 2024-11-28T06:56:38,773 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112802b21ca383cc4ae4b02858eb03b14456_d1f05a2e2429924e6c4e8d32d50aa9ca, store=[table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:38,773 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112802b21ca383cc4ae4b02858eb03b14456_d1f05a2e2429924e6c4e8d32d50aa9ca because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:38,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742312_1488 (size=4469) 2024-11-28T06:56:38,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-28T06:56:38,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:38,843 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing d1f05a2e2429924e6c4e8d32d50aa9ca 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T06:56:38,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=A 2024-11-28T06:56:38,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:38,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=B 2024-11-28T06:56:38,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:38,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=C 2024-11-28T06:56:38,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:38,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128baeef9cf866c49f487ee275511eb5bbd_d1f05a2e2429924e6c4e8d32d50aa9ca is 50, key is test_row_0/A:col10/1732776997456/Put/seqid=0 2024-11-28T06:56:38,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742313_1489 (size=12454) 2024-11-28T06:56:39,177 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/2eda224d6af34beabe1211f9e212f9b9 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/2eda224d6af34beabe1211f9e212f9b9 2024-11-28T06:56:39,182 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f05a2e2429924e6c4e8d32d50aa9ca/B of d1f05a2e2429924e6c4e8d32d50aa9ca into 2eda224d6af34beabe1211f9e212f9b9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:39,182 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:39,182 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., storeName=d1f05a2e2429924e6c4e8d32d50aa9ca/B, priority=13, startTime=1732776998734; duration=0sec 2024-11-28T06:56:39,182 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:56:39,182 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f05a2e2429924e6c4e8d32d50aa9ca:B 2024-11-28T06:56:39,182 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:56:39,185 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37109 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:56:39,185 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): d1f05a2e2429924e6c4e8d32d50aa9ca/C is initiating minor compaction (all files) 2024-11-28T06:56:39,185 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f05a2e2429924e6c4e8d32d50aa9ca/C in TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:39,185 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/56a61576bc4b4e939d4381491640816c, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/0cbb43648fac46cfa57c06a5f70c393c, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/139242ce9eb8452c8744cc6c47ae23c5] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp, totalSize=36.2 K 2024-11-28T06:56:39,186 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 56a61576bc4b4e939d4381491640816c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=767, earliestPutTs=1732776995160 2024-11-28T06:56:39,186 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cbb43648fac46cfa57c06a5f70c393c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=802, earliestPutTs=1732776995225 2024-11-28T06:56:39,187 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 139242ce9eb8452c8744cc6c47ae23c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=817, earliestPutTs=1732776997195 2024-11-28T06:56:39,192 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f05a2e2429924e6c4e8d32d50aa9ca#A#compaction#410 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:39,192 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/1822d7a504d34202a229b6e45caa83be is 175, key is test_row_0/A:col10/1732776997407/Put/seqid=0 2024-11-28T06:56:39,205 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f05a2e2429924e6c4e8d32d50aa9ca#C#compaction#412 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:39,205 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/b089a10e369449acb270bb71a96121b4 is 50, key is test_row_0/C:col10/1732776997407/Put/seqid=0 2024-11-28T06:56:39,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742314_1490 (size=31563) 2024-11-28T06:56:39,218 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/1822d7a504d34202a229b6e45caa83be as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/1822d7a504d34202a229b6e45caa83be 2024-11-28T06:56:39,225 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f05a2e2429924e6c4e8d32d50aa9ca/A of d1f05a2e2429924e6c4e8d32d50aa9ca into 1822d7a504d34202a229b6e45caa83be(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:39,225 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:39,225 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., storeName=d1f05a2e2429924e6c4e8d32d50aa9ca/A, priority=13, startTime=1732776998734; duration=0sec 2024-11-28T06:56:39,225 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:39,225 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f05a2e2429924e6c4e8d32d50aa9ca:A 2024-11-28T06:56:39,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742315_1491 (size=12609) 2024-11-28T06:56:39,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:39,313 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128baeef9cf866c49f487ee275511eb5bbd_d1f05a2e2429924e6c4e8d32d50aa9ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128baeef9cf866c49f487ee275511eb5bbd_d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:39,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/74ddcecb2bdb4ba49187980a08c64a8d, store: [table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:39,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/74ddcecb2bdb4ba49187980a08c64a8d is 175, key is test_row_0/A:col10/1732776997456/Put/seqid=0 2024-11-28T06:56:39,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:39,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:39,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742316_1492 (size=31255) 2024-11-28T06:56:39,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:39,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42208 deadline: 1732777059396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:39,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:39,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:39,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:39,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:39,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42230 deadline: 1732777059396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:39,405 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:39,405 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:39,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:39,405 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:39,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42204 deadline: 1732777059397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:39,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:39,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:39,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:39,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:39,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42208 deadline: 1732777059516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:39,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:39,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42230 deadline: 1732777059516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:39,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:39,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42204 deadline: 1732777059516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:39,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:39,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:39,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:39,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:39,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777059626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:39,627 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_1' in d1f05a2e2429924e6c4e8d32d50aa9ca of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:56:39,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:39,628 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:39,628 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:39,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:39,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777059635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:39,638 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_0' in d1f05a2e2429924e6c4e8d32d50aa9ca of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:56:39,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:39,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:39,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:39,646 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/b089a10e369449acb270bb71a96121b4 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/b089a10e369449acb270bb71a96121b4 2024-11-28T06:56:39,652 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f05a2e2429924e6c4e8d32d50aa9ca/C of d1f05a2e2429924e6c4e8d32d50aa9ca into b089a10e369449acb270bb71a96121b4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:39,652 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:39,652 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., storeName=d1f05a2e2429924e6c4e8d32d50aa9ca/C, priority=13, startTime=1732776998734; duration=0sec 2024-11-28T06:56:39,652 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:39,652 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f05a2e2429924e6c4e8d32d50aa9ca:C 2024-11-28T06:56:39,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:39,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42208 deadline: 1732777059725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:39,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:39,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:39,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:39,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:39,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42230 deadline: 1732777059725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:39,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:39,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:39,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:39,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:39,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42204 deadline: 1732777059725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:39,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:39,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:39,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:39,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-28T06:56:39,772 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=837, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/74ddcecb2bdb4ba49187980a08c64a8d 2024-11-28T06:56:39,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/3eedd143af5e409b8d79700cd4e4f3fc is 50, key is test_row_0/B:col10/1732776997456/Put/seqid=0 2024-11-28T06:56:39,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742317_1493 (size=12301) 2024-11-28T06:56:40,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42230 deadline: 1732777060036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:40,039 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:40,039 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:40,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:40,039 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42204 deadline: 1732777060036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:40,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:40,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:40,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:40,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42208 deadline: 1732777060037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:40,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:40,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:40,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:40,192 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=837 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/3eedd143af5e409b8d79700cd4e4f3fc 2024-11-28T06:56:40,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/5202e0be8e2647ad82e0346e2d3975e6 is 50, key is test_row_0/C:col10/1732776997456/Put/seqid=0 2024-11-28T06:56:40,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742318_1494 (size=12301) 2024-11-28T06:56:40,221 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=837 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/5202e0be8e2647ad82e0346e2d3975e6 2024-11-28T06:56:40,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/74ddcecb2bdb4ba49187980a08c64a8d as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/74ddcecb2bdb4ba49187980a08c64a8d 2024-11-28T06:56:40,230 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/74ddcecb2bdb4ba49187980a08c64a8d, entries=150, sequenceid=837, filesize=30.5 K 2024-11-28T06:56:40,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/3eedd143af5e409b8d79700cd4e4f3fc as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/3eedd143af5e409b8d79700cd4e4f3fc 2024-11-28T06:56:40,235 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/3eedd143af5e409b8d79700cd4e4f3fc, entries=150, sequenceid=837, filesize=12.0 K 2024-11-28T06:56:40,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/5202e0be8e2647ad82e0346e2d3975e6 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/5202e0be8e2647ad82e0346e2d3975e6 2024-11-28T06:56:40,241 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/5202e0be8e2647ad82e0346e2d3975e6, entries=150, sequenceid=837, filesize=12.0 K 2024-11-28T06:56:40,242 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for d1f05a2e2429924e6c4e8d32d50aa9ca in 1399ms, sequenceid=837, compaction requested=false 2024-11-28T06:56:40,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:40,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:40,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-28T06:56:40,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-28T06:56:40,245 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-28T06:56:40,245 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6240 sec 2024-11-28T06:56:40,248 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 2.6290 sec 2024-11-28T06:56:40,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:40,549 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1f05a2e2429924e6c4e8d32d50aa9ca 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T06:56:40,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=A 2024-11-28T06:56:40,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:40,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=B 2024-11-28T06:56:40,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:40,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=C 2024-11-28T06:56:40,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:40,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411280fad66ecacf04c159fd795eea3225a12_d1f05a2e2429924e6c4e8d32d50aa9ca is 50, key is test_row_0/A:col10/1732776999393/Put/seqid=0 2024-11-28T06:56:40,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742319_1495 (size=14994) 2024-11-28T06:56:40,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:40,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42208 deadline: 1732777060591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:40,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:40,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:40,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:40,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:40,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42204 deadline: 1732777060597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:40,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:40,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:40,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:40,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:40,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42230 deadline: 1732777060597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:40,602 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:40,602 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:40,602 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:40,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:40,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42208 deadline: 1732777060706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:40,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:40,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42204 deadline: 1732777060706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:40,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:40,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42230 deadline: 1732777060706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:40,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:40,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:40,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:40,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:40,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42204 deadline: 1732777060916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:40,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:40,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:40,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:40,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:40,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42208 deadline: 1732777060916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:40,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:40,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:40,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:40,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:40,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42230 deadline: 1732777060916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:40,920 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:40,920 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:40,920 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:40,977 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:40,981 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411280fad66ecacf04c159fd795eea3225a12_d1f05a2e2429924e6c4e8d32d50aa9ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411280fad66ecacf04c159fd795eea3225a12_d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:40,982 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/211b1fa0e8f64579b30594763a24c0bf, store: [table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:40,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/211b1fa0e8f64579b30594763a24c0bf is 175, key is test_row_0/A:col10/1732776999393/Put/seqid=0 2024-11-28T06:56:40,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742320_1496 (size=39949) 2024-11-28T06:56:41,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:41,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42204 deadline: 1732777061225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:41,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:41,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:41,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42208 deadline: 1732777061225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:41,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:41,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:41,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:41,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:41,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42230 deadline: 1732777061225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:41,229 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:41,229 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:41,229 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:41,388 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=854, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/211b1fa0e8f64579b30594763a24c0bf 2024-11-28T06:56:41,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/4e6bad083a46494ba0dcdc53e3e1f9c4 is 50, key is test_row_0/B:col10/1732776999393/Put/seqid=0 2024-11-28T06:56:41,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742321_1497 (size=12301) 2024-11-28T06:56:41,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=854 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/4e6bad083a46494ba0dcdc53e3e1f9c4 2024-11-28T06:56:41,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/cef56bae01ef413caad559ef179ea659 is 50, key is test_row_0/C:col10/1732776999393/Put/seqid=0 2024-11-28T06:56:41,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742322_1498 (size=12301) 2024-11-28T06:56:41,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=854 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/cef56bae01ef413caad559ef179ea659 2024-11-28T06:56:41,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/211b1fa0e8f64579b30594763a24c0bf as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/211b1fa0e8f64579b30594763a24c0bf 2024-11-28T06:56:41,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/211b1fa0e8f64579b30594763a24c0bf, entries=200, sequenceid=854, filesize=39.0 K 2024-11-28T06:56:41,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/4e6bad083a46494ba0dcdc53e3e1f9c4 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/4e6bad083a46494ba0dcdc53e3e1f9c4 2024-11-28T06:56:41,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/4e6bad083a46494ba0dcdc53e3e1f9c4, entries=150, sequenceid=854, filesize=12.0 K 2024-11-28T06:56:41,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/cef56bae01ef413caad559ef179ea659 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/cef56bae01ef413caad559ef179ea659 2024-11-28T06:56:41,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/cef56bae01ef413caad559ef179ea659, entries=150, sequenceid=854, filesize=12.0 K 2024-11-28T06:56:41,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for d1f05a2e2429924e6c4e8d32d50aa9ca in 929ms, sequenceid=854, compaction requested=true 2024-11-28T06:56:41,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:41,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f05a2e2429924e6c4e8d32d50aa9ca:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:56:41,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:41,477 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:56:41,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f05a2e2429924e6c4e8d32d50aa9ca:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:56:41,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:41,477 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:56:41,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f05a2e2429924e6c4e8d32d50aa9ca:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:56:41,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:56:41,477 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102767 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:56:41,477 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37211 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:56:41,478 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): d1f05a2e2429924e6c4e8d32d50aa9ca/A is initiating minor compaction (all files) 2024-11-28T06:56:41,478 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): d1f05a2e2429924e6c4e8d32d50aa9ca/B is initiating minor compaction (all files) 2024-11-28T06:56:41,478 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f05a2e2429924e6c4e8d32d50aa9ca/A in TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:41,478 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f05a2e2429924e6c4e8d32d50aa9ca/B in TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:41,478 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/1822d7a504d34202a229b6e45caa83be, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/74ddcecb2bdb4ba49187980a08c64a8d, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/211b1fa0e8f64579b30594763a24c0bf] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp, totalSize=100.4 K 2024-11-28T06:56:41,478 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/2eda224d6af34beabe1211f9e212f9b9, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/3eedd143af5e409b8d79700cd4e4f3fc, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/4e6bad083a46494ba0dcdc53e3e1f9c4] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp, totalSize=36.3 K 2024-11-28T06:56:41,478 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:41,478 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. files: [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/1822d7a504d34202a229b6e45caa83be, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/74ddcecb2bdb4ba49187980a08c64a8d, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/211b1fa0e8f64579b30594763a24c0bf] 2024-11-28T06:56:41,478 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 2eda224d6af34beabe1211f9e212f9b9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=817, earliestPutTs=1732776997195 2024-11-28T06:56:41,478 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 3eedd143af5e409b8d79700cd4e4f3fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=837, earliestPutTs=1732776997456 2024-11-28T06:56:41,478 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1822d7a504d34202a229b6e45caa83be, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=817, earliestPutTs=1732776997195 2024-11-28T06:56:41,479 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e6bad083a46494ba0dcdc53e3e1f9c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=854, earliestPutTs=1732776999393 2024-11-28T06:56:41,479 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74ddcecb2bdb4ba49187980a08c64a8d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=837, earliestPutTs=1732776997456 2024-11-28T06:56:41,479 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 211b1fa0e8f64579b30594763a24c0bf, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=854, earliestPutTs=1732776999393 2024-11-28T06:56:41,486 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f05a2e2429924e6c4e8d32d50aa9ca#B#compaction#418 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:41,486 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/014fbd16d2ef46fcbdd57b148802f28e is 50, key is test_row_0/B:col10/1732776999393/Put/seqid=0 2024-11-28T06:56:41,491 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:41,502 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411289154e13a61834cb292931d01979bf151_d1f05a2e2429924e6c4e8d32d50aa9ca store=[table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:41,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742323_1499 (size=12711) 2024-11-28T06:56:41,504 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411289154e13a61834cb292931d01979bf151_d1f05a2e2429924e6c4e8d32d50aa9ca, store=[table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:41,504 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411289154e13a61834cb292931d01979bf151_d1f05a2e2429924e6c4e8d32d50aa9ca because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:41,511 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/014fbd16d2ef46fcbdd57b148802f28e as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/014fbd16d2ef46fcbdd57b148802f28e 2024-11-28T06:56:41,515 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f05a2e2429924e6c4e8d32d50aa9ca/B of d1f05a2e2429924e6c4e8d32d50aa9ca into 014fbd16d2ef46fcbdd57b148802f28e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:41,515 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:41,515 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., storeName=d1f05a2e2429924e6c4e8d32d50aa9ca/B, priority=13, startTime=1732777001477; duration=0sec 2024-11-28T06:56:41,515 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:56:41,515 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f05a2e2429924e6c4e8d32d50aa9ca:B 2024-11-28T06:56:41,515 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:56:41,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742324_1500 (size=4469) 2024-11-28T06:56:41,516 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f05a2e2429924e6c4e8d32d50aa9ca#A#compaction#419 average throughput is 0.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:41,517 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/5af6921b43114a91b77d71f1663a9999 is 175, key is test_row_0/A:col10/1732776999393/Put/seqid=0 2024-11-28T06:56:41,517 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37211 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:56:41,517 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): d1f05a2e2429924e6c4e8d32d50aa9ca/C is initiating minor compaction (all files) 2024-11-28T06:56:41,517 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f05a2e2429924e6c4e8d32d50aa9ca/C in TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:41,517 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/b089a10e369449acb270bb71a96121b4, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/5202e0be8e2647ad82e0346e2d3975e6, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/cef56bae01ef413caad559ef179ea659] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp, totalSize=36.3 K 2024-11-28T06:56:41,518 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting b089a10e369449acb270bb71a96121b4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=817, earliestPutTs=1732776997195 2024-11-28T06:56:41,518 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 5202e0be8e2647ad82e0346e2d3975e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=837, earliestPutTs=1732776997456 2024-11-28T06:56:41,518 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting cef56bae01ef413caad559ef179ea659, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=854, earliestPutTs=1732776999393 2024-11-28T06:56:41,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742325_1501 (size=31665) 2024-11-28T06:56:41,530 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/5af6921b43114a91b77d71f1663a9999 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/5af6921b43114a91b77d71f1663a9999 2024-11-28T06:56:41,535 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f05a2e2429924e6c4e8d32d50aa9ca/A of d1f05a2e2429924e6c4e8d32d50aa9ca into 5af6921b43114a91b77d71f1663a9999(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:41,535 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:41,535 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., storeName=d1f05a2e2429924e6c4e8d32d50aa9ca/A, priority=13, startTime=1732777001477; duration=0sec 2024-11-28T06:56:41,535 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:41,535 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f05a2e2429924e6c4e8d32d50aa9ca:A 2024-11-28T06:56:41,540 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f05a2e2429924e6c4e8d32d50aa9ca#C#compaction#420 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:41,541 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/44140d3ec05c46db9c1cab22a58aada6 is 50, key is test_row_0/C:col10/1732776999393/Put/seqid=0 2024-11-28T06:56:41,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742326_1502 (size=12711) 2024-11-28T06:56:41,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:41,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1f05a2e2429924e6c4e8d32d50aa9ca 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T06:56:41,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=A 2024-11-28T06:56:41,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:41,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=B 2024-11-28T06:56:41,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:41,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=C 2024-11-28T06:56:41,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:41,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411288fb0dbe8bd8e4d7da1ca806023cc47ab_d1f05a2e2429924e6c4e8d32d50aa9ca is 50, key is test_row_0/A:col10/1732777000588/Put/seqid=0 2024-11-28T06:56:41,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742327_1503 (size=14994) 2024-11-28T06:56:41,663 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:41,665 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411288fb0dbe8bd8e4d7da1ca806023cc47ab_d1f05a2e2429924e6c4e8d32d50aa9ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288fb0dbe8bd8e4d7da1ca806023cc47ab_d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:41,666 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/307d0458579845df8afecc3a51efe1a7, store: [table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:41,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/307d0458579845df8afecc3a51efe1a7 is 175, key is test_row_0/A:col10/1732777000588/Put/seqid=0 2024-11-28T06:56:41,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742328_1504 (size=39949) 2024-11-28T06:56:41,673 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=876, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/307d0458579845df8afecc3a51efe1a7 2024-11-28T06:56:41,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/1fb4d72f230a4947adca9a7223ad39d1 is 50, key is test_row_0/B:col10/1732777000588/Put/seqid=0 2024-11-28T06:56:41,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742329_1505 (size=12301) 2024-11-28T06:56:41,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=876 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/1fb4d72f230a4947adca9a7223ad39d1 2024-11-28T06:56:41,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:41,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777061684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:41,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:41,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:41,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:41,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:41,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777061684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:41,690 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:41,690 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:41,690 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:41,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/be0c571792444c5dbe5920aca9842723 is 50, key is test_row_0/C:col10/1732777000588/Put/seqid=0 2024-11-28T06:56:41,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742330_1506 (size=12301) 2024-11-28T06:56:41,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=876 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/be0c571792444c5dbe5920aca9842723 2024-11-28T06:56:41,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/307d0458579845df8afecc3a51efe1a7 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/307d0458579845df8afecc3a51efe1a7 2024-11-28T06:56:41,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/307d0458579845df8afecc3a51efe1a7, entries=200, sequenceid=876, filesize=39.0 K 2024-11-28T06:56:41,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/1fb4d72f230a4947adca9a7223ad39d1 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/1fb4d72f230a4947adca9a7223ad39d1 2024-11-28T06:56:41,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/1fb4d72f230a4947adca9a7223ad39d1, entries=150, sequenceid=876, filesize=12.0 K 2024-11-28T06:56:41,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/be0c571792444c5dbe5920aca9842723 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/be0c571792444c5dbe5920aca9842723 2024-11-28T06:56:41,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/be0c571792444c5dbe5920aca9842723, entries=150, sequenceid=876, filesize=12.0 K 2024-11-28T06:56:41,735 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for d1f05a2e2429924e6c4e8d32d50aa9ca in 86ms, sequenceid=876, compaction requested=false 2024-11-28T06:56:41,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:41,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:41,742 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1f05a2e2429924e6c4e8d32d50aa9ca 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T06:56:41,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=A 2024-11-28T06:56:41,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:41,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=B 2024-11-28T06:56:41,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:41,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=C 2024-11-28T06:56:41,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:41,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112852860c6d27074d6ab891ba507153101b_d1f05a2e2429924e6c4e8d32d50aa9ca is 50, key is test_row_0/A:col10/1732777001741/Put/seqid=0 2024-11-28T06:56:41,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742331_1507 (size=14994) 2024-11-28T06:56:41,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-28T06:56:41,776 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:56:41,777 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:56:41,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-28T06:56:41,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:41,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-28T06:56:41,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42230 deadline: 1732777061773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:41,779 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:41,779 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:41,779 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:41,779 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:56:41,780 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:56:41,780 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:56:41,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:41,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42204 deadline: 1732777061777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:41,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:41,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42208 deadline: 1732777061779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:41,784 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:41,784 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:41,784 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:41,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:41,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:41,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:41,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:41,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777061795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:41,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:41,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:41,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:41,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:41,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777061795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:41,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:41,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:41,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:41,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-28T06:56:41,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:41,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42230 deadline: 1732777061886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:41,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:41,888 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:41,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42204 deadline: 1732777061886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:41,888 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:41,888 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:41,889 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:41,889 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:41,889 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:41,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:41,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42208 deadline: 1732777061896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:41,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:41,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:41,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:41,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-28T06:56:41,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:41,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:41,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:41,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:41,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:41,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:41,974 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/44140d3ec05c46db9c1cab22a58aada6 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/44140d3ec05c46db9c1cab22a58aada6 2024-11-28T06:56:41,981 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f05a2e2429924e6c4e8d32d50aa9ca/C of d1f05a2e2429924e6c4e8d32d50aa9ca into 44140d3ec05c46db9c1cab22a58aada6(size=12.4 K), total size for store is 24.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:41,981 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:41,981 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., storeName=d1f05a2e2429924e6c4e8d32d50aa9ca/C, priority=13, startTime=1732777001477; duration=0sec 2024-11-28T06:56:41,981 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:41,981 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f05a2e2429924e6c4e8d32d50aa9ca:C 2024-11-28T06:56:42,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:42,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777062006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:42,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:42,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777062006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:42,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:42,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:42,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:42,008 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:42,008 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:42,008 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:42,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-28T06:56:42,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:42,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:42,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:42,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:42,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:42,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:42,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-28T06:56:42,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:42,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42230 deadline: 1732777062096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:42,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:42,100 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:42,100 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:42,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42204 deadline: 1732777062096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:42,100 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:42,100 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:42,100 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:42,100 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:42,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:42,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42208 deadline: 1732777062105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:42,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:42,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:42,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:42,156 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:42,160 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112852860c6d27074d6ab891ba507153101b_d1f05a2e2429924e6c4e8d32d50aa9ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112852860c6d27074d6ab891ba507153101b_d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:42,162 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/7009b19b0b6047bcb26b5a91d359b8fc, store: [table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:42,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/7009b19b0b6047bcb26b5a91d359b8fc is 175, key is test_row_0/A:col10/1732777001741/Put/seqid=0 2024-11-28T06:56:42,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742332_1508 (size=39949) 2024-11-28T06:56:42,188 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=889, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/7009b19b0b6047bcb26b5a91d359b8fc 2024-11-28T06:56:42,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/7d10aa71624e445d84e672113b372901 is 50, key is test_row_0/B:col10/1732777001741/Put/seqid=0 2024-11-28T06:56:42,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742333_1509 (size=12301) 2024-11-28T06:56:42,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=889 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/7d10aa71624e445d84e672113b372901 2024-11-28T06:56:42,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/09130cb7593a4907a691a61647234b4a is 50, key is test_row_0/C:col10/1732777001741/Put/seqid=0 2024-11-28T06:56:42,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-28T06:56:42,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:42,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:42,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:42,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:42,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:42,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:42,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742334_1510 (size=12301) 2024-11-28T06:56:42,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=889 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/09130cb7593a4907a691a61647234b4a 2024-11-28T06:56:42,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/7009b19b0b6047bcb26b5a91d359b8fc as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/7009b19b0b6047bcb26b5a91d359b8fc 2024-11-28T06:56:42,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/7009b19b0b6047bcb26b5a91d359b8fc, entries=200, sequenceid=889, filesize=39.0 K 2024-11-28T06:56:42,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/7d10aa71624e445d84e672113b372901 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/7d10aa71624e445d84e672113b372901 2024-11-28T06:56:42,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/7d10aa71624e445d84e672113b372901, entries=150, sequenceid=889, filesize=12.0 K 2024-11-28T06:56:42,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/09130cb7593a4907a691a61647234b4a as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/09130cb7593a4907a691a61647234b4a 2024-11-28T06:56:42,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/09130cb7593a4907a691a61647234b4a, entries=150, sequenceid=889, filesize=12.0 K 2024-11-28T06:56:42,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for d1f05a2e2429924e6c4e8d32d50aa9ca in 541ms, sequenceid=889, compaction requested=true 2024-11-28T06:56:42,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:42,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f05a2e2429924e6c4e8d32d50aa9ca:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:56:42,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:42,283 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:56:42,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f05a2e2429924e6c4e8d32d50aa9ca:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:56:42,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:42,283 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:56:42,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f05a2e2429924e6c4e8d32d50aa9ca:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:56:42,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:56:42,284 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111563 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:56:42,284 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:56:42,284 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): d1f05a2e2429924e6c4e8d32d50aa9ca/A is initiating minor compaction (all files) 2024-11-28T06:56:42,284 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): d1f05a2e2429924e6c4e8d32d50aa9ca/B is initiating minor compaction (all files) 2024-11-28T06:56:42,284 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f05a2e2429924e6c4e8d32d50aa9ca/A in TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:42,284 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f05a2e2429924e6c4e8d32d50aa9ca/B in TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:42,284 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/5af6921b43114a91b77d71f1663a9999, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/307d0458579845df8afecc3a51efe1a7, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/7009b19b0b6047bcb26b5a91d359b8fc] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp, totalSize=108.9 K 2024-11-28T06:56:42,284 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/014fbd16d2ef46fcbdd57b148802f28e, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/1fb4d72f230a4947adca9a7223ad39d1, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/7d10aa71624e445d84e672113b372901] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp, totalSize=36.4 K 2024-11-28T06:56:42,284 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:42,285 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. files: [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/5af6921b43114a91b77d71f1663a9999, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/307d0458579845df8afecc3a51efe1a7, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/7009b19b0b6047bcb26b5a91d359b8fc] 2024-11-28T06:56:42,287 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5af6921b43114a91b77d71f1663a9999, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=854, earliestPutTs=1732776999393 2024-11-28T06:56:42,287 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 014fbd16d2ef46fcbdd57b148802f28e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=854, earliestPutTs=1732776999393 2024-11-28T06:56:42,287 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 307d0458579845df8afecc3a51efe1a7, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=876, earliestPutTs=1732777000588 2024-11-28T06:56:42,287 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fb4d72f230a4947adca9a7223ad39d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=876, earliestPutTs=1732777000588 2024-11-28T06:56:42,287 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7009b19b0b6047bcb26b5a91d359b8fc, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=889, earliestPutTs=1732777001670 2024-11-28T06:56:42,288 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d10aa71624e445d84e672113b372901, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=889, earliestPutTs=1732777001670 2024-11-28T06:56:42,295 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:42,297 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f05a2e2429924e6c4e8d32d50aa9ca#B#compaction#428 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:42,297 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/fcdb7b129fdf4e9ba9ec2eed9e7d8cd6 is 50, key is test_row_0/B:col10/1732777001741/Put/seqid=0 2024-11-28T06:56:42,311 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128e9d35e9da6cb406d8774f26d41231bc1_d1f05a2e2429924e6c4e8d32d50aa9ca store=[table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:42,313 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128e9d35e9da6cb406d8774f26d41231bc1_d1f05a2e2429924e6c4e8d32d50aa9ca, store=[table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:42,313 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128e9d35e9da6cb406d8774f26d41231bc1_d1f05a2e2429924e6c4e8d32d50aa9ca because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:42,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:42,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1f05a2e2429924e6c4e8d32d50aa9ca 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T06:56:42,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=A 2024-11-28T06:56:42,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:42,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=B 2024-11-28T06:56:42,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:42,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=C 2024-11-28T06:56:42,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:42,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742335_1511 (size=12813) 2024-11-28T06:56:42,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742336_1512 (size=4469) 2024-11-28T06:56:42,378 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f05a2e2429924e6c4e8d32d50aa9ca#A#compaction#427 average throughput is 0.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:42,379 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/6b8c1ebbd46d40c1bddb7d63316b8531 is 175, key is test_row_0/A:col10/1732777001741/Put/seqid=0 2024-11-28T06:56:42,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:42,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777062379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:42,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:42,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777062379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:42,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-28T06:56:42,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:42,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:42,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:42,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:42,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:42,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:42,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-28T06:56:42,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:42,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 274 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42230 deadline: 1732777062406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:42,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:42,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42204 deadline: 1732777062406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:42,411 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:42,412 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:42,412 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:42,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:42,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:42,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:42,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:42,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42208 deadline: 1732777062415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:42,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:42,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:42,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:42,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128b03aeac207944024b1bb935203d80330_d1f05a2e2429924e6c4e8d32d50aa9ca is 50, key is test_row_0/A:col10/1732777001771/Put/seqid=0 2024-11-28T06:56:42,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742337_1513 (size=31767) 2024-11-28T06:56:42,437 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/6b8c1ebbd46d40c1bddb7d63316b8531 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/6b8c1ebbd46d40c1bddb7d63316b8531 2024-11-28T06:56:42,441 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f05a2e2429924e6c4e8d32d50aa9ca/A of d1f05a2e2429924e6c4e8d32d50aa9ca into 6b8c1ebbd46d40c1bddb7d63316b8531(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:42,441 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:42,441 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., storeName=d1f05a2e2429924e6c4e8d32d50aa9ca/A, priority=13, startTime=1732777002283; duration=0sec 2024-11-28T06:56:42,441 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:56:42,441 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f05a2e2429924e6c4e8d32d50aa9ca:A 2024-11-28T06:56:42,441 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:56:42,442 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:56:42,442 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): d1f05a2e2429924e6c4e8d32d50aa9ca/C is initiating minor compaction (all files) 2024-11-28T06:56:42,442 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f05a2e2429924e6c4e8d32d50aa9ca/C in TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:42,442 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/44140d3ec05c46db9c1cab22a58aada6, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/be0c571792444c5dbe5920aca9842723, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/09130cb7593a4907a691a61647234b4a] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp, totalSize=36.4 K 2024-11-28T06:56:42,443 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44140d3ec05c46db9c1cab22a58aada6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=854, earliestPutTs=1732776999393 2024-11-28T06:56:42,443 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting be0c571792444c5dbe5920aca9842723, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=876, earliestPutTs=1732777000588 2024-11-28T06:56:42,443 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09130cb7593a4907a691a61647234b4a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=889, earliestPutTs=1732777001670 2024-11-28T06:56:42,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742338_1514 (size=12454) 2024-11-28T06:56:42,477 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f05a2e2429924e6c4e8d32d50aa9ca#C#compaction#430 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:42,477 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/dc657c83eec34ac79d07eddf935ee04d is 50, key is test_row_0/C:col10/1732777001741/Put/seqid=0 2024-11-28T06:56:42,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:42,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777062496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:42,502 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:42,502 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:42,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:42,502 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:42,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777062496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:42,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:42,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:42,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:42,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742339_1515 (size=12813) 2024-11-28T06:56:42,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-28T06:56:42,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:42,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:42,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:42,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:42,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:42,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:42,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-28T06:56:42,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:42,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:42,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:42,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:42,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:42,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:42,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:42,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777062706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:42,710 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:42,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:42,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:42,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:42,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777062716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:42,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:42,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:42,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:42,746 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/fcdb7b129fdf4e9ba9ec2eed9e7d8cd6 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/fcdb7b129fdf4e9ba9ec2eed9e7d8cd6 2024-11-28T06:56:42,752 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f05a2e2429924e6c4e8d32d50aa9ca/B of d1f05a2e2429924e6c4e8d32d50aa9ca into fcdb7b129fdf4e9ba9ec2eed9e7d8cd6(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:42,752 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:42,752 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., storeName=d1f05a2e2429924e6c4e8d32d50aa9ca/B, priority=13, startTime=1732777002283; duration=0sec 2024-11-28T06:56:42,753 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:42,753 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f05a2e2429924e6c4e8d32d50aa9ca:B 2024-11-28T06:56:42,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-28T06:56:42,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:42,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:42,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:42,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:42,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:42,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:42,865 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:42,868 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128b03aeac207944024b1bb935203d80330_d1f05a2e2429924e6c4e8d32d50aa9ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128b03aeac207944024b1bb935203d80330_d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:42,869 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/b26d83630eee4331afa61bf197c88283, store: [table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:42,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/b26d83630eee4331afa61bf197c88283 is 175, key is test_row_0/A:col10/1732777001771/Put/seqid=0 2024-11-28T06:56:42,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742340_1516 (size=31255) 2024-11-28T06:56:42,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-28T06:56:42,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:42,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42204 deadline: 1732777062915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:42,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:42,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42230 deadline: 1732777062915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:42,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:42,918 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:42,918 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:42,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:42,918 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:42,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:42,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:42,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42208 deadline: 1732777062925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:42,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:42,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:42,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:42,934 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/dc657c83eec34ac79d07eddf935ee04d as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/dc657c83eec34ac79d07eddf935ee04d 2024-11-28T06:56:42,940 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f05a2e2429924e6c4e8d32d50aa9ca/C of d1f05a2e2429924e6c4e8d32d50aa9ca into dc657c83eec34ac79d07eddf935ee04d(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:42,940 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:42,940 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., storeName=d1f05a2e2429924e6c4e8d32d50aa9ca/C, priority=13, startTime=1732777002283; duration=0sec 2024-11-28T06:56:42,940 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:42,940 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f05a2e2429924e6c4e8d32d50aa9ca:C 2024-11-28T06:56:43,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-28T06:56:43,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:43,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:43,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:43,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:43,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:43,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:43,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:43,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777063015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:43,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:43,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:43,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:43,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:43,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777063027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:43,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:43,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:43,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:43,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-28T06:56:43,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:43,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:43,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:43,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:43,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:43,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:43,273 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=910, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/b26d83630eee4331afa61bf197c88283 2024-11-28T06:56:43,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/c0c29124504a4fb4b70bc87e7f6e50d5 is 50, key is test_row_0/B:col10/1732777001771/Put/seqid=0 2024-11-28T06:56:43,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742341_1517 (size=12301) 2024-11-28T06:56:43,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=910 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/c0c29124504a4fb4b70bc87e7f6e50d5 2024-11-28T06:56:43,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-28T06:56:43,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:43,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:43,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:43,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:43,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:43,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:43,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/aafa45e04731459eac3079da5cd8e66d is 50, key is test_row_0/C:col10/1732777001771/Put/seqid=0 2024-11-28T06:56:43,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742342_1518 (size=12301) 2024-11-28T06:56:43,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=910 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/aafa45e04731459eac3079da5cd8e66d 2024-11-28T06:56:43,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/b26d83630eee4331afa61bf197c88283 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/b26d83630eee4331afa61bf197c88283 2024-11-28T06:56:43,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/b26d83630eee4331afa61bf197c88283, entries=150, sequenceid=910, filesize=30.5 K 2024-11-28T06:56:43,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/c0c29124504a4fb4b70bc87e7f6e50d5 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/c0c29124504a4fb4b70bc87e7f6e50d5 2024-11-28T06:56:43,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/c0c29124504a4fb4b70bc87e7f6e50d5, entries=150, sequenceid=910, filesize=12.0 K 2024-11-28T06:56:43,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/aafa45e04731459eac3079da5cd8e66d as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/aafa45e04731459eac3079da5cd8e66d 2024-11-28T06:56:43,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/aafa45e04731459eac3079da5cd8e66d, entries=150, sequenceid=910, filesize=12.0 K 2024-11-28T06:56:43,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for d1f05a2e2429924e6c4e8d32d50aa9ca in 1029ms, sequenceid=910, compaction requested=false 2024-11-28T06:56:43,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:43,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-28T06:56:43,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:43,463 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing d1f05a2e2429924e6c4e8d32d50aa9ca 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T06:56:43,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=A 2024-11-28T06:56:43,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:43,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=B 2024-11-28T06:56:43,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:43,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=C 2024-11-28T06:56:43,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:43,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112867ca3cc22b1c40db93f31264c5d75459_d1f05a2e2429924e6c4e8d32d50aa9ca is 50, key is test_row_0/A:col10/1732777002375/Put/seqid=0 2024-11-28T06:56:43,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742343_1519 (size=12454) 2024-11-28T06:56:43,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:43,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:43,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:43,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777063579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:43,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:43,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:43,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777063580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:43,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:43,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:43,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:43,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:43,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:43,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777063685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:43,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777063685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:43,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:43,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:43,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:43,869 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T06:56:43,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:43,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777063895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:43,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:43,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:43,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:43,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:43,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777063895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:43,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:43,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:43,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:43,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:43,918 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112867ca3cc22b1c40db93f31264c5d75459_d1f05a2e2429924e6c4e8d32d50aa9ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112867ca3cc22b1c40db93f31264c5d75459_d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:43,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/8ef56060d57a46ddaa86b308a2e28726, store: [table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:43,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/8ef56060d57a46ddaa86b308a2e28726 is 175, key is test_row_0/A:col10/1732777002375/Put/seqid=0 2024-11-28T06:56:43,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742344_1520 (size=31255) 2024-11-28T06:56:43,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-28T06:56:43,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:43,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 276 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42230 deadline: 1732777063935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:43,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:43,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42204 deadline: 1732777063935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:43,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:43,938 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_1' in d1f05a2e2429924e6c4e8d32d50aa9ca of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:56:43,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42208 deadline: 1732777063935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:43,939 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:43,939 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:43,939 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_1' in d1f05a2e2429924e6c4e8d32d50aa9ca of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:56:43,939 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:43,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:43,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:43,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:43,939 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_1' in d1f05a2e2429924e6c4e8d32d50aa9ca of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:56:43,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:43,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:43,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:44,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:44,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777064205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:44,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:44,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:44,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:44,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:44,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777064205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:44,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:44,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:44,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:44,328 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=926, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/8ef56060d57a46ddaa86b308a2e28726 2024-11-28T06:56:44,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/39208b99952e4fe0bbe4c0e72aca9a7f is 50, key is test_row_0/B:col10/1732777002375/Put/seqid=0 2024-11-28T06:56:44,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742345_1521 (size=12301) 2024-11-28T06:56:44,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:44,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777064716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:44,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:44,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:44,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:44,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:44,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777064716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:44,718 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:44,718 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:44,718 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:44,746 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=926 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/39208b99952e4fe0bbe4c0e72aca9a7f 2024-11-28T06:56:44,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/b1d278ef46934b94af6cc3f9da22e8a5 is 50, key is test_row_0/C:col10/1732777002375/Put/seqid=0 2024-11-28T06:56:44,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742346_1522 (size=12301) 2024-11-28T06:56:45,162 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=926 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/b1d278ef46934b94af6cc3f9da22e8a5 2024-11-28T06:56:45,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/8ef56060d57a46ddaa86b308a2e28726 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/8ef56060d57a46ddaa86b308a2e28726 2024-11-28T06:56:45,169 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/8ef56060d57a46ddaa86b308a2e28726, entries=150, sequenceid=926, filesize=30.5 K 2024-11-28T06:56:45,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/39208b99952e4fe0bbe4c0e72aca9a7f as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/39208b99952e4fe0bbe4c0e72aca9a7f 2024-11-28T06:56:45,172 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/39208b99952e4fe0bbe4c0e72aca9a7f, entries=150, sequenceid=926, filesize=12.0 K 2024-11-28T06:56:45,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/b1d278ef46934b94af6cc3f9da22e8a5 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/b1d278ef46934b94af6cc3f9da22e8a5 2024-11-28T06:56:45,176 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/b1d278ef46934b94af6cc3f9da22e8a5, entries=150, sequenceid=926, filesize=12.0 K 2024-11-28T06:56:45,176 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for d1f05a2e2429924e6c4e8d32d50aa9ca in 1713ms, sequenceid=926, compaction requested=true 2024-11-28T06:56:45,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:45,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:45,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-28T06:56:45,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-28T06:56:45,179 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-28T06:56:45,179 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3970 sec 2024-11-28T06:56:45,180 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 3.4030 sec 2024-11-28T06:56:45,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:45,727 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1f05a2e2429924e6c4e8d32d50aa9ca 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-28T06:56:45,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=A 2024-11-28T06:56:45,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:56:45,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=B 2024-11-28T06:56:45,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:56:45,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=C 2024-11-28T06:56:45,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T06:56:45,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112864037e0b55704ebb9a915a75d88329ca_d1f05a2e2429924e6c4e8d32d50aa9ca is 50, key is test_row_0/A:col10/1732777005726/Put/seqid=0 2024-11-28T06:56:45,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742347_1523 (size=14994) 2024-11-28T06:56:45,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:45,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777065745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:45,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:45,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:45,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:45,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:45,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777065751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:45,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:45,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:45,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:45,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:45,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777065856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:45,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:45,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:45,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:45,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:45,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777065865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:45,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:45,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:45,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:45,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-28T06:56:45,946 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:56:45,946 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:56:45,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-28T06:56:45,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-28T06:56:45,948 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:56:45,948 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:56:45,948 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:56:45,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:45,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42230 deadline: 1732777065955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:45,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:45,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42208 deadline: 1732777065955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:45,957 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_1' in d1f05a2e2429924e6c4e8d32d50aa9ca of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:56:45,958 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:45,958 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:45,958 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:45,958 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_1' in d1f05a2e2429924e6c4e8d32d50aa9ca of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:56:45,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:45,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:45,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:45,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:45,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42204 deadline: 1732777065965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:45,968 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_1' in d1f05a2e2429924e6c4e8d32d50aa9ca of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:56:45,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:45,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:45,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:46,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-28T06:56:46,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:46,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777066065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:46,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:46,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777066075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:46,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:46,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:46,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:46,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-28T06:56:46,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:46,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:46,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:46,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:46,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:46,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:46,138 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:46,140 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112864037e0b55704ebb9a915a75d88329ca_d1f05a2e2429924e6c4e8d32d50aa9ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112864037e0b55704ebb9a915a75d88329ca_d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:46,141 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/0768d01a122e41698dc0704d20b4e703, store: [table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:46,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/0768d01a122e41698dc0704d20b4e703 is 175, key is test_row_0/A:col10/1732777005726/Put/seqid=0 2024-11-28T06:56:46,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742348_1524 (size=39949) 2024-11-28T06:56:46,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-28T06:56:46,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:46,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:46,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:46,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:46,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:46,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:46,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-28T06:56:46,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:46,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777066376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:46,378 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:46,378 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:46,378 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:46,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:46,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777066385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:46,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:46,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:46,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:46,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-28T06:56:46,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:46,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:46,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:46,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:46,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:46,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:46,544 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=947, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/0768d01a122e41698dc0704d20b4e703 2024-11-28T06:56:46,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/e5318b020f6d4fcdbbfeefd5f1c8ac68 is 50, key is test_row_0/B:col10/1732777005726/Put/seqid=0 2024-11-28T06:56:46,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742349_1525 (size=12301) 2024-11-28T06:56:46,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-28T06:56:46,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:46,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:46,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:46,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:46,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:46,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:46,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-28T06:56:46,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-28T06:56:46,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:46,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:46,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:46,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:46,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:46,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:46,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-28T06:56:46,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:46,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:46,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:46,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:46,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:46,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:46,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:46,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777066885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:46,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:46,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:46,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777066896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:46,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:46,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:46,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:46,954 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=947 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/e5318b020f6d4fcdbbfeefd5f1c8ac68 2024-11-28T06:56:46,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/1efde5ef6f524b048c9afde0041d93d8 is 50, key is test_row_0/C:col10/1732777005726/Put/seqid=0 2024-11-28T06:56:46,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742350_1526 (size=12301) 2024-11-28T06:56:47,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-28T06:56:47,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:47,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:47,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:47,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:47,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:47,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:47,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-28T06:56:47,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-28T06:56:47,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:47,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:47,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:47,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:47,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:47,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:47,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-28T06:56:47,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:47,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:47,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:47,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:47,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:47,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:47,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=947 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/1efde5ef6f524b048c9afde0041d93d8 2024-11-28T06:56:47,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/0768d01a122e41698dc0704d20b4e703 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/0768d01a122e41698dc0704d20b4e703 2024-11-28T06:56:47,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/0768d01a122e41698dc0704d20b4e703, entries=200, sequenceid=947, filesize=39.0 K 2024-11-28T06:56:47,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/e5318b020f6d4fcdbbfeefd5f1c8ac68 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/e5318b020f6d4fcdbbfeefd5f1c8ac68 2024-11-28T06:56:47,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/e5318b020f6d4fcdbbfeefd5f1c8ac68, entries=150, sequenceid=947, filesize=12.0 K 2024-11-28T06:56:47,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/1efde5ef6f524b048c9afde0041d93d8 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/1efde5ef6f524b048c9afde0041d93d8 2024-11-28T06:56:47,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/1efde5ef6f524b048c9afde0041d93d8, entries=150, sequenceid=947, filesize=12.0 K 2024-11-28T06:56:47,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for d1f05a2e2429924e6c4e8d32d50aa9ca in 1667ms, sequenceid=947, compaction requested=true 2024-11-28T06:56:47,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:47,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f05a2e2429924e6c4e8d32d50aa9ca:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:56:47,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:47,395 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T06:56:47,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f05a2e2429924e6c4e8d32d50aa9ca:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:56:47,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:56:47,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f05a2e2429924e6c4e8d32d50aa9ca:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:56:47,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-28T06:56:47,395 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T06:56:47,397 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134226 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T06:56:47,398 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): d1f05a2e2429924e6c4e8d32d50aa9ca/A is initiating minor compaction (all files) 2024-11-28T06:56:47,398 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f05a2e2429924e6c4e8d32d50aa9ca/A in TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:47,398 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/6b8c1ebbd46d40c1bddb7d63316b8531, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/b26d83630eee4331afa61bf197c88283, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/8ef56060d57a46ddaa86b308a2e28726, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/0768d01a122e41698dc0704d20b4e703] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp, totalSize=131.1 K 2024-11-28T06:56:47,398 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:47,398 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. files: [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/6b8c1ebbd46d40c1bddb7d63316b8531, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/b26d83630eee4331afa61bf197c88283, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/8ef56060d57a46ddaa86b308a2e28726, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/0768d01a122e41698dc0704d20b4e703] 2024-11-28T06:56:47,398 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49716 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T06:56:47,398 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): d1f05a2e2429924e6c4e8d32d50aa9ca/B is initiating minor compaction (all files) 2024-11-28T06:56:47,398 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f05a2e2429924e6c4e8d32d50aa9ca/B in TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:47,398 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/fcdb7b129fdf4e9ba9ec2eed9e7d8cd6, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/c0c29124504a4fb4b70bc87e7f6e50d5, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/39208b99952e4fe0bbe4c0e72aca9a7f, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/e5318b020f6d4fcdbbfeefd5f1c8ac68] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp, totalSize=48.6 K 2024-11-28T06:56:47,399 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting fcdb7b129fdf4e9ba9ec2eed9e7d8cd6, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=889, earliestPutTs=1732777001670 2024-11-28T06:56:47,399 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b8c1ebbd46d40c1bddb7d63316b8531, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=889, earliestPutTs=1732777001670 2024-11-28T06:56:47,399 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting b26d83630eee4331afa61bf197c88283, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=910, earliestPutTs=1732777001770 2024-11-28T06:56:47,399 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting c0c29124504a4fb4b70bc87e7f6e50d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=910, earliestPutTs=1732777001770 2024-11-28T06:56:47,399 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ef56060d57a46ddaa86b308a2e28726, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=926, earliestPutTs=1732777002365 2024-11-28T06:56:47,400 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 39208b99952e4fe0bbe4c0e72aca9a7f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=926, earliestPutTs=1732777002365 2024-11-28T06:56:47,400 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0768d01a122e41698dc0704d20b4e703, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=947, earliestPutTs=1732777003533 2024-11-28T06:56:47,400 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting e5318b020f6d4fcdbbfeefd5f1c8ac68, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=947, earliestPutTs=1732777003533 2024-11-28T06:56:47,406 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:47,407 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411288d38b135b07948ea912806a0e31665ef_d1f05a2e2429924e6c4e8d32d50aa9ca store=[table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:47,408 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f05a2e2429924e6c4e8d32d50aa9ca#B#compaction#439 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:47,408 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/cd2ae974dc6048e89021d188d00b04fe is 50, key is test_row_0/B:col10/1732777005726/Put/seqid=0 2024-11-28T06:56:47,409 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411288d38b135b07948ea912806a0e31665ef_d1f05a2e2429924e6c4e8d32d50aa9ca, store=[table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:47,409 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411288d38b135b07948ea912806a0e31665ef_d1f05a2e2429924e6c4e8d32d50aa9ca because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:47,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742352_1528 (size=12949) 2024-11-28T06:56:47,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742351_1527 (size=4469) 2024-11-28T06:56:47,423 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f05a2e2429924e6c4e8d32d50aa9ca#A#compaction#440 average throughput is 1.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:47,423 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/298d7a4b4e66484091c1ea6f1f1fc1be is 175, key is test_row_0/A:col10/1732777005726/Put/seqid=0 2024-11-28T06:56:47,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742353_1529 (size=31903) 2024-11-28T06:56:47,431 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/298d7a4b4e66484091c1ea6f1f1fc1be as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/298d7a4b4e66484091c1ea6f1f1fc1be 2024-11-28T06:56:47,435 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d1f05a2e2429924e6c4e8d32d50aa9ca/A of d1f05a2e2429924e6c4e8d32d50aa9ca into 298d7a4b4e66484091c1ea6f1f1fc1be(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:47,435 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:47,435 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., storeName=d1f05a2e2429924e6c4e8d32d50aa9ca/A, priority=12, startTime=1732777007395; duration=0sec 2024-11-28T06:56:47,435 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:56:47,435 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f05a2e2429924e6c4e8d32d50aa9ca:A 2024-11-28T06:56:47,435 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T06:56:47,436 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49716 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T06:56:47,436 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): d1f05a2e2429924e6c4e8d32d50aa9ca/C is initiating minor compaction (all files) 2024-11-28T06:56:47,436 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f05a2e2429924e6c4e8d32d50aa9ca/C in TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:47,436 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/dc657c83eec34ac79d07eddf935ee04d, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/aafa45e04731459eac3079da5cd8e66d, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/b1d278ef46934b94af6cc3f9da22e8a5, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/1efde5ef6f524b048c9afde0041d93d8] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp, totalSize=48.6 K 2024-11-28T06:56:47,436 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc657c83eec34ac79d07eddf935ee04d, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=889, earliestPutTs=1732777001670 2024-11-28T06:56:47,436 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting aafa45e04731459eac3079da5cd8e66d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=910, earliestPutTs=1732777001770 2024-11-28T06:56:47,437 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1d278ef46934b94af6cc3f9da22e8a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=926, earliestPutTs=1732777002365 2024-11-28T06:56:47,437 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1efde5ef6f524b048c9afde0041d93d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=947, earliestPutTs=1732777003533 2024-11-28T06:56:47,443 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f05a2e2429924e6c4e8d32d50aa9ca#C#compaction#441 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:47,444 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/c315c40d15e245549ae0ff46a3936b65 is 50, key is test_row_0/C:col10/1732777005726/Put/seqid=0 2024-11-28T06:56:47,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742354_1530 (size=12949) 2024-11-28T06:56:47,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-28T06:56:47,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:47,473 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing d1f05a2e2429924e6c4e8d32d50aa9ca 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T06:56:47,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=A 2024-11-28T06:56:47,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:47,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=B 2024-11-28T06:56:47,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:47,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=C 2024-11-28T06:56:47,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:47,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128e72f298667604eb4afa68a1d610e3a1e_d1f05a2e2429924e6c4e8d32d50aa9ca is 50, key is test_row_0/A:col10/1732777005743/Put/seqid=0 2024-11-28T06:56:47,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742355_1531 (size=12454) 2024-11-28T06:56:47,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:47,484 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128e72f298667604eb4afa68a1d610e3a1e_d1f05a2e2429924e6c4e8d32d50aa9ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e72f298667604eb4afa68a1d610e3a1e_d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:47,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/3b81434bec5745308524551f03bb67cc, store: [table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:47,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/3b81434bec5745308524551f03bb67cc is 175, key is test_row_0/A:col10/1732777005743/Put/seqid=0 2024-11-28T06:56:47,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742356_1532 (size=31255) 2024-11-28T06:56:47,489 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=959, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/3b81434bec5745308524551f03bb67cc 2024-11-28T06:56:47,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/fd2ff998accf416db7087e62f6f7e793 is 50, key is test_row_0/B:col10/1732777005743/Put/seqid=0 2024-11-28T06:56:47,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742357_1533 (size=12301) 2024-11-28T06:56:47,500 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=959 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/fd2ff998accf416db7087e62f6f7e793 2024-11-28T06:56:47,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/c6676be22ab24643848b8e3a032e366c is 50, key is test_row_0/C:col10/1732777005743/Put/seqid=0 2024-11-28T06:56:47,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742358_1534 (size=12301) 2024-11-28T06:56:47,826 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/cd2ae974dc6048e89021d188d00b04fe as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/cd2ae974dc6048e89021d188d00b04fe 2024-11-28T06:56:47,831 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d1f05a2e2429924e6c4e8d32d50aa9ca/B of d1f05a2e2429924e6c4e8d32d50aa9ca into cd2ae974dc6048e89021d188d00b04fe(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:47,831 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:47,831 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., storeName=d1f05a2e2429924e6c4e8d32d50aa9ca/B, priority=12, startTime=1732777007395; duration=0sec 2024-11-28T06:56:47,831 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:47,831 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f05a2e2429924e6c4e8d32d50aa9ca:B 2024-11-28T06:56:47,855 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/c315c40d15e245549ae0ff46a3936b65 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/c315c40d15e245549ae0ff46a3936b65 2024-11-28T06:56:47,861 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d1f05a2e2429924e6c4e8d32d50aa9ca/C of d1f05a2e2429924e6c4e8d32d50aa9ca into c315c40d15e245549ae0ff46a3936b65(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:47,861 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:47,861 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., storeName=d1f05a2e2429924e6c4e8d32d50aa9ca/C, priority=12, startTime=1732777007395; duration=0sec 2024-11-28T06:56:47,862 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:47,862 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f05a2e2429924e6c4e8d32d50aa9ca:C 2024-11-28T06:56:47,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:47,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:47,911 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=959 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/c6676be22ab24643848b8e3a032e366c 2024-11-28T06:56:47,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/3b81434bec5745308524551f03bb67cc as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/3b81434bec5745308524551f03bb67cc 2024-11-28T06:56:47,920 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/3b81434bec5745308524551f03bb67cc, entries=150, sequenceid=959, filesize=30.5 K 2024-11-28T06:56:47,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/fd2ff998accf416db7087e62f6f7e793 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/fd2ff998accf416db7087e62f6f7e793 2024-11-28T06:56:47,927 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/fd2ff998accf416db7087e62f6f7e793, entries=150, sequenceid=959, filesize=12.0 K 2024-11-28T06:56:47,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/c6676be22ab24643848b8e3a032e366c as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/c6676be22ab24643848b8e3a032e366c 2024-11-28T06:56:47,933 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/c6676be22ab24643848b8e3a032e366c, entries=150, sequenceid=959, filesize=12.0 K 2024-11-28T06:56:47,934 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=80.51 KB/82440 for d1f05a2e2429924e6c4e8d32d50aa9ca in 461ms, sequenceid=959, compaction requested=false 2024-11-28T06:56:47,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:47,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:47,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-28T06:56:47,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-28T06:56:47,937 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-28T06:56:47,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:47,937 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9870 sec 2024-11-28T06:56:47,937 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1f05a2e2429924e6c4e8d32d50aa9ca 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-28T06:56:47,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=A 2024-11-28T06:56:47,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:47,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=B 2024-11-28T06:56:47,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:47,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=C 2024-11-28T06:56:47,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:47,939 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.9920 sec 2024-11-28T06:56:47,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128e982ad4cc96e4473830721b5e0807a47_d1f05a2e2429924e6c4e8d32d50aa9ca is 50, key is test_row_0/A:col10/1732777007930/Put/seqid=0 2024-11-28T06:56:47,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742359_1535 (size=14994) 2024-11-28T06:56:47,964 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:47,967 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128e982ad4cc96e4473830721b5e0807a47_d1f05a2e2429924e6c4e8d32d50aa9ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e982ad4cc96e4473830721b5e0807a47_d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:47,967 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/b876587aa6a046d1ae32c9226284c9fc, store: [table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:47,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/b876587aa6a046d1ae32c9226284c9fc is 175, key is test_row_0/A:col10/1732777007930/Put/seqid=0 2024-11-28T06:56:47,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742360_1536 (size=39949) 2024-11-28T06:56:47,975 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=977, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/b876587aa6a046d1ae32c9226284c9fc 2024-11-28T06:56:47,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/87f1446066214f089f2d5d5ac7be7468 is 50, key is test_row_0/B:col10/1732777007930/Put/seqid=0 2024-11-28T06:56:48,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742361_1537 (size=12301) 2024-11-28T06:56:48,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:48,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777068004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:48,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:48,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:48,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:48,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:48,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777068004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:48,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:48,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:48,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:48,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-28T06:56:48,095 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:56:48,096 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T06:56:48,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-28T06:56:48,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-28T06:56:48,098 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T06:56:48,099 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T06:56:48,099 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T06:56:48,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:48,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777068116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:48,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:48,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:48,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:48,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:48,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777068116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:48,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:48,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:48,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:48,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-28T06:56:48,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-28T06:56:48,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:48,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:48,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:48,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:48,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:48,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:48,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:48,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777068325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:48,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:48,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:48,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777068325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:48,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:48,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:48,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:48,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:48,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:48,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=977 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/87f1446066214f089f2d5d5ac7be7468 2024-11-28T06:56:48,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-28T06:56:48,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:48,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:48,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:48,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:48,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:48,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:48,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-28T06:56:48,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/8f9caea9c8f548ed9288f35e69131adb is 50, key is test_row_0/C:col10/1732777007930/Put/seqid=0 2024-11-28T06:56:48,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742362_1538 (size=12301) 2024-11-28T06:56:48,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-28T06:56:48,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:48,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:48,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:48,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:48,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:48,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:48,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:48,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777068636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:48,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:48,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:48,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:48,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:48,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777068636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:48,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:48,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:48,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:48,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-28T06:56:48,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:48,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:48,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:48,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:48,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:48,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:48,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-28T06:56:48,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=977 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/8f9caea9c8f548ed9288f35e69131adb 2024-11-28T06:56:48,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/b876587aa6a046d1ae32c9226284c9fc as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/b876587aa6a046d1ae32c9226284c9fc 2024-11-28T06:56:48,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/b876587aa6a046d1ae32c9226284c9fc, entries=200, sequenceid=977, filesize=39.0 K 2024-11-28T06:56:48,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/87f1446066214f089f2d5d5ac7be7468 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/87f1446066214f089f2d5d5ac7be7468 2024-11-28T06:56:48,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/87f1446066214f089f2d5d5ac7be7468, entries=150, sequenceid=977, filesize=12.0 K 2024-11-28T06:56:48,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/8f9caea9c8f548ed9288f35e69131adb as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/8f9caea9c8f548ed9288f35e69131adb 2024-11-28T06:56:48,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/8f9caea9c8f548ed9288f35e69131adb, entries=150, sequenceid=977, filesize=12.0 K 2024-11-28T06:56:48,860 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for d1f05a2e2429924e6c4e8d32d50aa9ca in 923ms, sequenceid=977, compaction requested=true 2024-11-28T06:56:48,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:48,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f05a2e2429924e6c4e8d32d50aa9ca:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:56:48,860 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:56:48,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:48,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f05a2e2429924e6c4e8d32d50aa9ca:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:56:48,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:48,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f05a2e2429924e6c4e8d32d50aa9ca:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:56:48,860 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:56:48,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:56:48,861 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:56:48,861 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:56:48,861 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): d1f05a2e2429924e6c4e8d32d50aa9ca/A is initiating minor compaction (all files) 2024-11-28T06:56:48,861 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): d1f05a2e2429924e6c4e8d32d50aa9ca/B is initiating minor compaction (all files) 2024-11-28T06:56:48,861 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f05a2e2429924e6c4e8d32d50aa9ca/A in TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:48,861 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f05a2e2429924e6c4e8d32d50aa9ca/B in TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:48,861 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/298d7a4b4e66484091c1ea6f1f1fc1be, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/3b81434bec5745308524551f03bb67cc, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/b876587aa6a046d1ae32c9226284c9fc] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp, totalSize=100.7 K 2024-11-28T06:56:48,861 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/cd2ae974dc6048e89021d188d00b04fe, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/fd2ff998accf416db7087e62f6f7e793, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/87f1446066214f089f2d5d5ac7be7468] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp, totalSize=36.7 K 2024-11-28T06:56:48,861 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:48,861 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. files: [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/298d7a4b4e66484091c1ea6f1f1fc1be, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/3b81434bec5745308524551f03bb67cc, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/b876587aa6a046d1ae32c9226284c9fc] 2024-11-28T06:56:48,862 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting cd2ae974dc6048e89021d188d00b04fe, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=947, earliestPutTs=1732777003533 2024-11-28T06:56:48,862 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 298d7a4b4e66484091c1ea6f1f1fc1be, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=947, earliestPutTs=1732777003533 2024-11-28T06:56:48,862 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting fd2ff998accf416db7087e62f6f7e793, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=959, earliestPutTs=1732777005728 2024-11-28T06:56:48,862 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b81434bec5745308524551f03bb67cc, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=959, earliestPutTs=1732777005728 2024-11-28T06:56:48,862 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 87f1446066214f089f2d5d5ac7be7468, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=977, earliestPutTs=1732777007930 2024-11-28T06:56:48,862 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting b876587aa6a046d1ae32c9226284c9fc, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=977, earliestPutTs=1732777007918 2024-11-28T06:56:48,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43341 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-28T06:56:48,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:48,864 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing d1f05a2e2429924e6c4e8d32d50aa9ca 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-28T06:56:48,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=A 2024-11-28T06:56:48,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:48,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=B 2024-11-28T06:56:48,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:48,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=C 2024-11-28T06:56:48,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:48,867 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:48,872 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f05a2e2429924e6c4e8d32d50aa9ca#B#compaction#449 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:48,873 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/ab2327e2780a44dfb5ea25b992badfa2 is 50, key is test_row_0/B:col10/1732777007930/Put/seqid=0 2024-11-28T06:56:48,875 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128f689090c39c34990bceeb0e5e0425cf0_d1f05a2e2429924e6c4e8d32d50aa9ca store=[table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:48,878 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128f689090c39c34990bceeb0e5e0425cf0_d1f05a2e2429924e6c4e8d32d50aa9ca, store=[table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:48,878 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128f689090c39c34990bceeb0e5e0425cf0_d1f05a2e2429924e6c4e8d32d50aa9ca because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:48,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411285da609cccffd43f190436219d8f41a11_d1f05a2e2429924e6c4e8d32d50aa9ca is 50, key is test_row_0/A:col10/1732777007985/Put/seqid=0 2024-11-28T06:56:48,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742363_1539 (size=13051) 2024-11-28T06:56:48,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742364_1540 (size=4469) 2024-11-28T06:56:48,889 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f05a2e2429924e6c4e8d32d50aa9ca#A#compaction#448 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:48,889 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/b9dfa7b2c56c4f5997abe88a7bf3ee77 is 175, key is test_row_0/A:col10/1732777007930/Put/seqid=0 2024-11-28T06:56:48,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742365_1541 (size=12454) 2024-11-28T06:56:48,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742366_1542 (size=32005) 2024-11-28T06:56:49,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:49,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. as already flushing 2024-11-28T06:56:49,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:49,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777069178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:49,182 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:49,182 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:49,182 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:49,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:49,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777069179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:49,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:49,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:49,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:49,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-28T06:56:49,277 INFO [Thread-2070 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2070. 2024-11-28T06:56:49,277 DEBUG [Thread-2070 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:56:49,277 DEBUG [Thread-2070 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:49,277 DEBUG [Thread-2070 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:49,277 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:56:49,277 INFO [Thread-2068 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2068. 2024-11-28T06:56:49,277 DEBUG [Thread-2068 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:56:49,277 DEBUG [Thread-2068 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:49,277 DEBUG [Thread-2068 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:49,277 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:56:49,282 INFO [Thread-2064 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2064. 2024-11-28T06:56:49,282 DEBUG [Thread-2064 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:56:49,282 DEBUG [Thread-2064 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:49,282 INFO [Thread-2066 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2066. 2024-11-28T06:56:49,282 DEBUG [Thread-2064 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:49,282 DEBUG [Thread-2066 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:56:49,282 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:56:49,282 DEBUG [Thread-2066 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:49,282 DEBUG [Thread-2066 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:49,282 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:56:49,284 INFO [Thread-2062 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2062. 2024-11-28T06:56:49,284 DEBUG [Thread-2062 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:56:49,284 DEBUG [Thread-2062 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:49,284 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/ab2327e2780a44dfb5ea25b992badfa2 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/ab2327e2780a44dfb5ea25b992badfa2 2024-11-28T06:56:49,284 DEBUG [Thread-2062 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:49,284 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:56:49,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:49,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777069285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:49,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:49,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:49,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:49,289 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f05a2e2429924e6c4e8d32d50aa9ca/B of d1f05a2e2429924e6c4e8d32d50aa9ca into ab2327e2780a44dfb5ea25b992badfa2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:49,289 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:49,289 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., storeName=d1f05a2e2429924e6c4e8d32d50aa9ca/B, priority=13, startTime=1732777008860; duration=0sec 2024-11-28T06:56:49,289 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:56:49,289 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f05a2e2429924e6c4e8d32d50aa9ca:B 2024-11-28T06:56:49,289 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:56:49,290 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:56:49,290 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): d1f05a2e2429924e6c4e8d32d50aa9ca/C is initiating minor compaction (all files) 2024-11-28T06:56:49,290 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f05a2e2429924e6c4e8d32d50aa9ca/C in TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:49,290 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/c315c40d15e245549ae0ff46a3936b65, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/c6676be22ab24643848b8e3a032e366c, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/8f9caea9c8f548ed9288f35e69131adb] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp, totalSize=36.7 K 2024-11-28T06:56:49,290 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting c315c40d15e245549ae0ff46a3936b65, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=947, earliestPutTs=1732777003533 2024-11-28T06:56:49,291 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting c6676be22ab24643848b8e3a032e366c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=959, earliestPutTs=1732777005728 2024-11-28T06:56:49,291 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f9caea9c8f548ed9288f35e69131adb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=977, earliestPutTs=1732777007930 2024-11-28T06:56:49,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:49,296 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f05a2e2429924e6c4e8d32d50aa9ca#C#compaction#451 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:49,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777069296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:49,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:49,297 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/d6915d383f5f4991b7d5e97cfea89ac2 is 50, key is test_row_0/C:col10/1732777007930/Put/seqid=0 2024-11-28T06:56:49,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:49,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:49,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742367_1543 (size=13051) 2024-11-28T06:56:49,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:49,312 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411285da609cccffd43f190436219d8f41a11_d1f05a2e2429924e6c4e8d32d50aa9ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411285da609cccffd43f190436219d8f41a11_d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:49,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/102e03868a524a8aa3fede687e1c4ec0, store: [table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:49,313 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/b9dfa7b2c56c4f5997abe88a7bf3ee77 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/b9dfa7b2c56c4f5997abe88a7bf3ee77 2024-11-28T06:56:49,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/102e03868a524a8aa3fede687e1c4ec0 is 175, key is test_row_0/A:col10/1732777007985/Put/seqid=0 2024-11-28T06:56:49,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742368_1544 (size=31255) 2024-11-28T06:56:49,321 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f05a2e2429924e6c4e8d32d50aa9ca/A of d1f05a2e2429924e6c4e8d32d50aa9ca into b9dfa7b2c56c4f5997abe88a7bf3ee77(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:49,321 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:49,321 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., storeName=d1f05a2e2429924e6c4e8d32d50aa9ca/A, priority=13, startTime=1732777008860; duration=0sec 2024-11-28T06:56:49,321 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:49,321 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f05a2e2429924e6c4e8d32d50aa9ca:A 2024-11-28T06:56:49,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:49,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777069496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:49,497 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:49,497 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:49,497 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:49,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:49,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777069505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:49,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:49,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:49,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:49,704 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/d6915d383f5f4991b7d5e97cfea89ac2 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/d6915d383f5f4991b7d5e97cfea89ac2 2024-11-28T06:56:49,708 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f05a2e2429924e6c4e8d32d50aa9ca/C of d1f05a2e2429924e6c4e8d32d50aa9ca into d6915d383f5f4991b7d5e97cfea89ac2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:49,708 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:49,708 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., storeName=d1f05a2e2429924e6c4e8d32d50aa9ca/C, priority=13, startTime=1732777008860; duration=0sec 2024-11-28T06:56:49,708 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:49,708 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f05a2e2429924e6c4e8d32d50aa9ca:C 2024-11-28T06:56:49,720 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=994, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/102e03868a524a8aa3fede687e1c4ec0 2024-11-28T06:56:49,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/98c8ec51a817467baed05035c37242ae is 50, key is test_row_0/B:col10/1732777007985/Put/seqid=0 2024-11-28T06:56:49,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742369_1545 (size=12301) 2024-11-28T06:56:49,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:49,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777069806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:49,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:49,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:49,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:49,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:49,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777069816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:49,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:49,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:49,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:49,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:49,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42208 deadline: 1732777069975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:49,976 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_1' in d1f05a2e2429924e6c4e8d32d50aa9ca of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:56:49,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:49,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:49,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:49,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:49,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 278 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42230 deadline: 1732777069995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:49,996 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_1' in d1f05a2e2429924e6c4e8d32d50aa9ca of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:56:49,996 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:49,996 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:49,996 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:50,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:50,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42204 deadline: 1732777070006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:50,006 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 1cb9cf437a36,43341,1732776825568 for 'test_row_1' in d1f05a2e2429924e6c4e8d32d50aa9ca of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T06:56:50,007 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:50,007 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:50,007 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:50,130 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=994 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/98c8ec51a817467baed05035c37242ae 2024-11-28T06:56:50,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/1c0faa7e06ee41af9585cb1e9e3c43b2 is 50, key is test_row_0/C:col10/1732777007985/Put/seqid=0 2024-11-28T06:56:50,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742370_1546 (size=12301) 2024-11-28T06:56:50,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-28T06:56:50,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:50,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42240 deadline: 1732777070315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:50,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:50,317 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:50,317 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:50,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T06:56:50,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42192 deadline: 1732777070326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 2024-11-28T06:56:50,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 , the old value is region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:50,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T06:56:50,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., hostname=1cb9cf437a36,43341,1732776825568, seqNum=5 because the exception is null or not the one we care about 2024-11-28T06:56:50,550 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=994 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/1c0faa7e06ee41af9585cb1e9e3c43b2 2024-11-28T06:56:50,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/102e03868a524a8aa3fede687e1c4ec0 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/102e03868a524a8aa3fede687e1c4ec0 2024-11-28T06:56:50,562 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/102e03868a524a8aa3fede687e1c4ec0, entries=150, sequenceid=994, filesize=30.5 K 2024-11-28T06:56:50,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/98c8ec51a817467baed05035c37242ae as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/98c8ec51a817467baed05035c37242ae 2024-11-28T06:56:50,568 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/98c8ec51a817467baed05035c37242ae, entries=150, sequenceid=994, filesize=12.0 K 2024-11-28T06:56:50,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/1c0faa7e06ee41af9585cb1e9e3c43b2 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/1c0faa7e06ee41af9585cb1e9e3c43b2 2024-11-28T06:56:50,572 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/1c0faa7e06ee41af9585cb1e9e3c43b2, entries=150, sequenceid=994, filesize=12.0 K 2024-11-28T06:56:50,573 INFO [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for d1f05a2e2429924e6c4e8d32d50aa9ca in 1709ms, sequenceid=994, compaction requested=false 2024-11-28T06:56:50,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:50,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:50,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-28T06:56:50,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-28T06:56:50,577 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-28T06:56:50,577 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4760 sec 2024-11-28T06:56:50,579 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.4820 sec 2024-11-28T06:56:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43341 {}] regionserver.HRegion(8855): Flush requested on d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:51,328 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1f05a2e2429924e6c4e8d32d50aa9ca 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-28T06:56:51,329 INFO [Thread-2051 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2051. 2024-11-28T06:56:51,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=A 2024-11-28T06:56:51,329 DEBUG [Thread-2051 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:56:51,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:51,329 DEBUG [Thread-2051 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:51,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=B 2024-11-28T06:56:51,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:51,329 DEBUG [Thread-2051 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:51,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=C 2024-11-28T06:56:51,329 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:56:51,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:56:51,339 INFO [Thread-2053 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2053. 2024-11-28T06:56:51,339 DEBUG [Thread-2053 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:56:51,339 DEBUG [Thread-2053 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:51,339 DEBUG [Thread-2053 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:51,339 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:56:51,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411289d210142b0e44aa086f30123d0083c53_d1f05a2e2429924e6c4e8d32d50aa9ca is 50, key is test_row_0/A:col10/1732777009177/Put/seqid=0 2024-11-28T06:56:51,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742371_1547 (size=12454) 2024-11-28T06:56:51,743 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:51,749 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411289d210142b0e44aa086f30123d0083c53_d1f05a2e2429924e6c4e8d32d50aa9ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411289d210142b0e44aa086f30123d0083c53_d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:56:51,750 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/41fa76efb65d4837a060375f3994f6ee, store: [table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:51,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/41fa76efb65d4837a060375f3994f6ee is 175, key is test_row_0/A:col10/1732777009177/Put/seqid=0 2024-11-28T06:56:51,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742372_1548 (size=31255) 2024-11-28T06:56:52,157 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=1013, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/41fa76efb65d4837a060375f3994f6ee 2024-11-28T06:56:52,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/d7af1272260f49e6a10ddd6468e1010f is 50, key is test_row_0/B:col10/1732777009177/Put/seqid=0 2024-11-28T06:56:52,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742373_1549 (size=12301) 2024-11-28T06:56:52,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-28T06:56:52,266 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:56:52,571 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=1013 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/d7af1272260f49e6a10ddd6468e1010f 2024-11-28T06:56:52,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/8c7796f09858451f8b064ff85157ced2 is 50, key is test_row_0/C:col10/1732777009177/Put/seqid=0 2024-11-28T06:56:52,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742374_1550 (size=12301) 2024-11-28T06:56:52,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=1013 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/8c7796f09858451f8b064ff85157ced2 2024-11-28T06:56:52,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/41fa76efb65d4837a060375f3994f6ee as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/41fa76efb65d4837a060375f3994f6ee 2024-11-28T06:56:53,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/41fa76efb65d4837a060375f3994f6ee, entries=150, sequenceid=1013, filesize=30.5 K 2024-11-28T06:56:53,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/d7af1272260f49e6a10ddd6468e1010f as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/d7af1272260f49e6a10ddd6468e1010f 2024-11-28T06:56:53,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/d7af1272260f49e6a10ddd6468e1010f, entries=150, sequenceid=1013, filesize=12.0 K 2024-11-28T06:56:53,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/8c7796f09858451f8b064ff85157ced2 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/8c7796f09858451f8b064ff85157ced2 2024-11-28T06:56:53,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/8c7796f09858451f8b064ff85157ced2, entries=150, sequenceid=1013, filesize=12.0 K 2024-11-28T06:56:53,008 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=6.71 KB/6870 for d1f05a2e2429924e6c4e8d32d50aa9ca in 1680ms, sequenceid=1013, compaction requested=true 2024-11-28T06:56:53,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:53,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f05a2e2429924e6c4e8d32d50aa9ca:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T06:56:53,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:53,008 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:56:53,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f05a2e2429924e6c4e8d32d50aa9ca:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T06:56:53,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:53,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f05a2e2429924e6c4e8d32d50aa9ca:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T06:56:53,008 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:56:53,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:56:53,009 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94515 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:56:53,009 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:56:53,009 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1541): d1f05a2e2429924e6c4e8d32d50aa9ca/A is initiating minor compaction (all files) 2024-11-28T06:56:53,009 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): d1f05a2e2429924e6c4e8d32d50aa9ca/B is initiating minor compaction (all files) 2024-11-28T06:56:53,009 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f05a2e2429924e6c4e8d32d50aa9ca/A in TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:53,009 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f05a2e2429924e6c4e8d32d50aa9ca/B in TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:53,009 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/b9dfa7b2c56c4f5997abe88a7bf3ee77, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/102e03868a524a8aa3fede687e1c4ec0, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/41fa76efb65d4837a060375f3994f6ee] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp, totalSize=92.3 K 2024-11-28T06:56:53,009 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/ab2327e2780a44dfb5ea25b992badfa2, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/98c8ec51a817467baed05035c37242ae, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/d7af1272260f49e6a10ddd6468e1010f] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp, totalSize=36.8 K 2024-11-28T06:56:53,009 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:53,009 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. files: [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/b9dfa7b2c56c4f5997abe88a7bf3ee77, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/102e03868a524a8aa3fede687e1c4ec0, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/41fa76efb65d4837a060375f3994f6ee] 2024-11-28T06:56:53,009 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting ab2327e2780a44dfb5ea25b992badfa2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=977, earliestPutTs=1732777007930 2024-11-28T06:56:53,009 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9dfa7b2c56c4f5997abe88a7bf3ee77, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=977, earliestPutTs=1732777007930 2024-11-28T06:56:53,010 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 98c8ec51a817467baed05035c37242ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=994, earliestPutTs=1732777007962 2024-11-28T06:56:53,010 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 102e03868a524a8aa3fede687e1c4ec0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=994, earliestPutTs=1732777007962 2024-11-28T06:56:53,010 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting d7af1272260f49e6a10ddd6468e1010f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1013, earliestPutTs=1732777009176 2024-11-28T06:56:53,010 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41fa76efb65d4837a060375f3994f6ee, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=1013, earliestPutTs=1732777009176 2024-11-28T06:56:53,017 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:53,018 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f05a2e2429924e6c4e8d32d50aa9ca#B#compaction#457 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:53,019 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411289350c3b6948247fba8f7d76aaa6e0abe_d1f05a2e2429924e6c4e8d32d50aa9ca store=[table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:53,019 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/73d641bf9116417a9f495462b2509b9f is 50, key is test_row_0/B:col10/1732777009177/Put/seqid=0 2024-11-28T06:56:53,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742375_1551 (size=13153) 2024-11-28T06:56:53,023 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411289350c3b6948247fba8f7d76aaa6e0abe_d1f05a2e2429924e6c4e8d32d50aa9ca, store=[table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:53,023 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411289350c3b6948247fba8f7d76aaa6e0abe_d1f05a2e2429924e6c4e8d32d50aa9ca because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:56:53,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742376_1552 (size=4469) 2024-11-28T06:56:53,427 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/73d641bf9116417a9f495462b2509b9f as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/73d641bf9116417a9f495462b2509b9f 2024-11-28T06:56:53,427 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f05a2e2429924e6c4e8d32d50aa9ca#A#compaction#458 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:53,427 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/2fb963d75cb34883b33abcce64a465cb is 175, key is test_row_0/A:col10/1732777009177/Put/seqid=0 2024-11-28T06:56:53,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742377_1553 (size=32107) 2024-11-28T06:56:53,431 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f05a2e2429924e6c4e8d32d50aa9ca/B of d1f05a2e2429924e6c4e8d32d50aa9ca into 73d641bf9116417a9f495462b2509b9f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:53,431 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:53,431 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., storeName=d1f05a2e2429924e6c4e8d32d50aa9ca/B, priority=13, startTime=1732777013008; duration=0sec 2024-11-28T06:56:53,431 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T06:56:53,431 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f05a2e2429924e6c4e8d32d50aa9ca:B 2024-11-28T06:56:53,431 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T06:56:53,432 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T06:56:53,432 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1541): d1f05a2e2429924e6c4e8d32d50aa9ca/C is initiating minor compaction (all files) 2024-11-28T06:56:53,432 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f05a2e2429924e6c4e8d32d50aa9ca/C in TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:56:53,432 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/d6915d383f5f4991b7d5e97cfea89ac2, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/1c0faa7e06ee41af9585cb1e9e3c43b2, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/8c7796f09858451f8b064ff85157ced2] into tmpdir=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp, totalSize=36.8 K 2024-11-28T06:56:53,432 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting d6915d383f5f4991b7d5e97cfea89ac2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=977, earliestPutTs=1732777007930 2024-11-28T06:56:53,433 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c0faa7e06ee41af9585cb1e9e3c43b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=994, earliestPutTs=1732777007962 2024-11-28T06:56:53,433 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c7796f09858451f8b064ff85157ced2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1013, earliestPutTs=1732777009176 2024-11-28T06:56:53,439 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f05a2e2429924e6c4e8d32d50aa9ca#C#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T06:56:53,439 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/90fe4878403442f5871cfc0a2f65a768 is 50, key is test_row_0/C:col10/1732777009177/Put/seqid=0 2024-11-28T06:56:53,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742378_1554 (size=13153) 2024-11-28T06:56:53,834 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/2fb963d75cb34883b33abcce64a465cb as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/2fb963d75cb34883b33abcce64a465cb 2024-11-28T06:56:53,838 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f05a2e2429924e6c4e8d32d50aa9ca/A of d1f05a2e2429924e6c4e8d32d50aa9ca into 2fb963d75cb34883b33abcce64a465cb(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:53,838 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:53,838 INFO [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., storeName=d1f05a2e2429924e6c4e8d32d50aa9ca/A, priority=13, startTime=1732777013008; duration=0sec 2024-11-28T06:56:53,838 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:53,838 DEBUG [RS:0;1cb9cf437a36:43341-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f05a2e2429924e6c4e8d32d50aa9ca:A 2024-11-28T06:56:53,846 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/90fe4878403442f5871cfc0a2f65a768 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/90fe4878403442f5871cfc0a2f65a768 2024-11-28T06:56:53,850 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f05a2e2429924e6c4e8d32d50aa9ca/C of d1f05a2e2429924e6c4e8d32d50aa9ca into 90fe4878403442f5871cfc0a2f65a768(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T06:56:53,850 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f05a2e2429924e6c4e8d32d50aa9ca: 2024-11-28T06:56:53,850 INFO [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., storeName=d1f05a2e2429924e6c4e8d32d50aa9ca/C, priority=13, startTime=1732777013008; duration=0sec 2024-11-28T06:56:53,850 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T06:56:53,850 DEBUG [RS:0;1cb9cf437a36:43341-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f05a2e2429924e6c4e8d32d50aa9ca:C 2024-11-28T06:57:00,031 INFO [Thread-2057 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2057. 2024-11-28T06:57:00,031 DEBUG [Thread-2057 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:57:00,031 DEBUG [Thread-2057 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:57:00,031 DEBUG [Thread-2057 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:57:00,031 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:57:00,078 INFO [Thread-2059 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2059. 2024-11-28T06:57:00,078 DEBUG [Thread-2059 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:57:00,078 DEBUG [Thread-2059 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:57:00,078 DEBUG [Thread-2059 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:57:00,078 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:57:00,100 INFO [Thread-2055 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2055. 2024-11-28T06:57:00,100 DEBUG [Thread-2055 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T06:57:00,100 DEBUG [Thread-2055 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:57:00,100 DEBUG [Thread-2055 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:57:00,100 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:57:00,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-28T06:57:00,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 151 2024-11-28T06:57:00,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 166 2024-11-28T06:57:00,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 222 2024-11-28T06:57:00,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 251 2024-11-28T06:57:00,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 148 2024-11-28T06:57:00,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-28T06:57:00,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2892 2024-11-28T06:57:00,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2918 2024-11-28T06:57:00,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2827 2024-11-28T06:57:00,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2897 2024-11-28T06:57:00,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2915 2024-11-28T06:57:00,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-28T06:57:00,101 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T06:57:00,101 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T06:57:00,101 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:57:00,101 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:57:00,101 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:57:00,101 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-28T06:57:00,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-28T06:57:00,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-28T06:57:00,104 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732777020104"}]},"ts":"1732777020104"} 2024-11-28T06:57:00,105 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-28T06:57:00,105 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-28T06:57:00,106 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T06:57:00,107 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1f05a2e2429924e6c4e8d32d50aa9ca, UNASSIGN}] 2024-11-28T06:57:00,107 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=154, ppid=153, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1f05a2e2429924e6c4e8d32d50aa9ca, UNASSIGN 2024-11-28T06:57:00,108 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=154 updating hbase:meta row=d1f05a2e2429924e6c4e8d32d50aa9ca, regionState=CLOSING, regionLocation=1cb9cf437a36,43341,1732776825568 2024-11-28T06:57:00,109 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=154, ppid=153, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1f05a2e2429924e6c4e8d32d50aa9ca, UNASSIGN because future has completed 2024-11-28T06:57:00,109 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T06:57:00,109 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; CloseRegionProcedure d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568}] 2024-11-28T06:57:00,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-28T06:57:00,262 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] handler.UnassignRegionHandler(122): Close d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:57:00,262 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T06:57:00,262 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1722): Closing d1f05a2e2429924e6c4e8d32d50aa9ca, disabling compactions & flushes 2024-11-28T06:57:00,262 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:57:00,262 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:57:00,262 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. after waiting 0 ms 2024-11-28T06:57:00,262 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:57:00,262 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(2902): Flushing d1f05a2e2429924e6c4e8d32d50aa9ca 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-28T06:57:00,262 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=A 2024-11-28T06:57:00,262 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:57:00,262 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=B 2024-11-28T06:57:00,263 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:57:00,263 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f05a2e2429924e6c4e8d32d50aa9ca, store=C 2024-11-28T06:57:00,263 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T06:57:00,268 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128bc964ba4f44246ca8ae507bf6b0ef467_d1f05a2e2429924e6c4e8d32d50aa9ca is 50, key is test_row_0/A:col10/1732777011336/Put/seqid=0 2024-11-28T06:57:00,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742379_1555 (size=9914) 2024-11-28T06:57:00,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-28T06:57:00,673 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:57:00,682 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128bc964ba4f44246ca8ae507bf6b0ef467_d1f05a2e2429924e6c4e8d32d50aa9ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128bc964ba4f44246ca8ae507bf6b0ef467_d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:57:00,683 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/88dce33dd4e24825a87e4f7f0b38400e, store: [table=TestAcidGuarantees family=A region=d1f05a2e2429924e6c4e8d32d50aa9ca] 2024-11-28T06:57:00,683 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/88dce33dd4e24825a87e4f7f0b38400e is 175, key is test_row_0/A:col10/1732777011336/Put/seqid=0 2024-11-28T06:57:00,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742380_1556 (size=22561) 2024-11-28T06:57:00,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-28T06:57:01,088 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=1023, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/88dce33dd4e24825a87e4f7f0b38400e 2024-11-28T06:57:01,098 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/fb6aad0007744d5f8ecc2d6eece82191 is 50, key is test_row_0/B:col10/1732777011336/Put/seqid=0 2024-11-28T06:57:01,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742381_1557 (size=9857) 2024-11-28T06:57:01,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-28T06:57:01,504 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=1023 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/fb6aad0007744d5f8ecc2d6eece82191 2024-11-28T06:57:01,510 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/afad475e326d46fab2e93d0afbe77f3e is 50, key is test_row_0/C:col10/1732777011336/Put/seqid=0 2024-11-28T06:57:01,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742382_1558 (size=9857) 2024-11-28T06:57:01,915 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=1023 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/afad475e326d46fab2e93d0afbe77f3e 2024-11-28T06:57:01,921 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/A/88dce33dd4e24825a87e4f7f0b38400e as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/88dce33dd4e24825a87e4f7f0b38400e 2024-11-28T06:57:01,925 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/88dce33dd4e24825a87e4f7f0b38400e, entries=100, sequenceid=1023, filesize=22.0 K 2024-11-28T06:57:01,926 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/B/fb6aad0007744d5f8ecc2d6eece82191 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/fb6aad0007744d5f8ecc2d6eece82191 2024-11-28T06:57:01,930 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/fb6aad0007744d5f8ecc2d6eece82191, entries=100, sequenceid=1023, filesize=9.6 K 2024-11-28T06:57:01,931 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/.tmp/C/afad475e326d46fab2e93d0afbe77f3e as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/afad475e326d46fab2e93d0afbe77f3e 2024-11-28T06:57:01,935 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/afad475e326d46fab2e93d0afbe77f3e, entries=100, sequenceid=1023, filesize=9.6 K 2024-11-28T06:57:01,935 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for d1f05a2e2429924e6c4e8d32d50aa9ca in 1673ms, sequenceid=1023, compaction requested=false 2024-11-28T06:57:01,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/421b9da479de45efa00010683b0a2ca3, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/57d6480fea0b499497eaa7fe37479afb, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/a662c96cab5d4c8eb97aaba5ba16cf46, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/6838b68679db4f309cd6da111982fe0d, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/899e47f9a9c14af5b1860932d68af59e, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/df07587d9d3f4656ac2412fbd7bfc225, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/e7388049d2ca4e8b89848a278677bede, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/ca575823ea24400593133a4af98fd990, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/adb7b57d8efd4147913baf1dfc8f22c9, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/1822d7a504d34202a229b6e45caa83be, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/74ddcecb2bdb4ba49187980a08c64a8d, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/211b1fa0e8f64579b30594763a24c0bf, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/5af6921b43114a91b77d71f1663a9999, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/307d0458579845df8afecc3a51efe1a7, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/7009b19b0b6047bcb26b5a91d359b8fc, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/6b8c1ebbd46d40c1bddb7d63316b8531, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/b26d83630eee4331afa61bf197c88283, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/8ef56060d57a46ddaa86b308a2e28726, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/0768d01a122e41698dc0704d20b4e703, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/298d7a4b4e66484091c1ea6f1f1fc1be, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/3b81434bec5745308524551f03bb67cc, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/b876587aa6a046d1ae32c9226284c9fc, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/b9dfa7b2c56c4f5997abe88a7bf3ee77, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/102e03868a524a8aa3fede687e1c4ec0, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/41fa76efb65d4837a060375f3994f6ee] to archive 2024-11-28T06:57:01,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T06:57:01,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/421b9da479de45efa00010683b0a2ca3 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/421b9da479de45efa00010683b0a2ca3 2024-11-28T06:57:01,941 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/57d6480fea0b499497eaa7fe37479afb to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/57d6480fea0b499497eaa7fe37479afb 2024-11-28T06:57:01,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/a662c96cab5d4c8eb97aaba5ba16cf46 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/a662c96cab5d4c8eb97aaba5ba16cf46 2024-11-28T06:57:01,944 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/6838b68679db4f309cd6da111982fe0d to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/6838b68679db4f309cd6da111982fe0d 2024-11-28T06:57:01,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/899e47f9a9c14af5b1860932d68af59e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/899e47f9a9c14af5b1860932d68af59e 2024-11-28T06:57:01,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/df07587d9d3f4656ac2412fbd7bfc225 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/df07587d9d3f4656ac2412fbd7bfc225 2024-11-28T06:57:01,948 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/e7388049d2ca4e8b89848a278677bede to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/e7388049d2ca4e8b89848a278677bede 2024-11-28T06:57:01,950 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/ca575823ea24400593133a4af98fd990 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/ca575823ea24400593133a4af98fd990 2024-11-28T06:57:01,951 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/adb7b57d8efd4147913baf1dfc8f22c9 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/adb7b57d8efd4147913baf1dfc8f22c9 2024-11-28T06:57:01,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/1822d7a504d34202a229b6e45caa83be to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/1822d7a504d34202a229b6e45caa83be 2024-11-28T06:57:01,954 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/74ddcecb2bdb4ba49187980a08c64a8d to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/74ddcecb2bdb4ba49187980a08c64a8d 2024-11-28T06:57:01,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/211b1fa0e8f64579b30594763a24c0bf to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/211b1fa0e8f64579b30594763a24c0bf 2024-11-28T06:57:01,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/5af6921b43114a91b77d71f1663a9999 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/5af6921b43114a91b77d71f1663a9999 2024-11-28T06:57:01,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/307d0458579845df8afecc3a51efe1a7 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/307d0458579845df8afecc3a51efe1a7 2024-11-28T06:57:01,959 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/7009b19b0b6047bcb26b5a91d359b8fc to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/7009b19b0b6047bcb26b5a91d359b8fc 2024-11-28T06:57:01,960 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/6b8c1ebbd46d40c1bddb7d63316b8531 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/6b8c1ebbd46d40c1bddb7d63316b8531 2024-11-28T06:57:01,961 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/b26d83630eee4331afa61bf197c88283 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/b26d83630eee4331afa61bf197c88283 2024-11-28T06:57:01,962 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/8ef56060d57a46ddaa86b308a2e28726 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/8ef56060d57a46ddaa86b308a2e28726 2024-11-28T06:57:01,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/0768d01a122e41698dc0704d20b4e703 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/0768d01a122e41698dc0704d20b4e703 2024-11-28T06:57:01,964 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/298d7a4b4e66484091c1ea6f1f1fc1be to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/298d7a4b4e66484091c1ea6f1f1fc1be 2024-11-28T06:57:01,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/3b81434bec5745308524551f03bb67cc to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/3b81434bec5745308524551f03bb67cc 2024-11-28T06:57:01,966 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/b876587aa6a046d1ae32c9226284c9fc to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/b876587aa6a046d1ae32c9226284c9fc 2024-11-28T06:57:01,967 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/b9dfa7b2c56c4f5997abe88a7bf3ee77 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/b9dfa7b2c56c4f5997abe88a7bf3ee77 2024-11-28T06:57:01,968 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/102e03868a524a8aa3fede687e1c4ec0 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/102e03868a524a8aa3fede687e1c4ec0 2024-11-28T06:57:01,969 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/41fa76efb65d4837a060375f3994f6ee to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/41fa76efb65d4837a060375f3994f6ee 2024-11-28T06:57:01,970 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/b4e6362f3bf0453b9b50ed6981103ce6, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/b594710925724c028c3cd11780a38a2b, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/f89b89d6c42a433a8a74ee42ca16b94e, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/8f6fc64673ea4c92a8a82af15c50142e, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/2419ee7c3e424462bfeaf81814ddadf5, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/d20de09ef09b460492c9eade728aa14d, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/5403d6220f514f949b1f7e51e2b271e8, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/369cbc5925484469bb32ef6df9ed592f, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/2eda224d6af34beabe1211f9e212f9b9, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/0d329cd372204c2dbef84b23d3313bb1, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/3eedd143af5e409b8d79700cd4e4f3fc, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/014fbd16d2ef46fcbdd57b148802f28e, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/4e6bad083a46494ba0dcdc53e3e1f9c4, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/1fb4d72f230a4947adca9a7223ad39d1, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/fcdb7b129fdf4e9ba9ec2eed9e7d8cd6, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/7d10aa71624e445d84e672113b372901, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/c0c29124504a4fb4b70bc87e7f6e50d5, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/39208b99952e4fe0bbe4c0e72aca9a7f, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/cd2ae974dc6048e89021d188d00b04fe, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/e5318b020f6d4fcdbbfeefd5f1c8ac68, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/fd2ff998accf416db7087e62f6f7e793, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/ab2327e2780a44dfb5ea25b992badfa2, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/87f1446066214f089f2d5d5ac7be7468, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/98c8ec51a817467baed05035c37242ae, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/d7af1272260f49e6a10ddd6468e1010f] to archive 2024-11-28T06:57:01,971 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T06:57:01,972 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/b4e6362f3bf0453b9b50ed6981103ce6 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/b4e6362f3bf0453b9b50ed6981103ce6 2024-11-28T06:57:01,973 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/b594710925724c028c3cd11780a38a2b to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/b594710925724c028c3cd11780a38a2b 2024-11-28T06:57:01,974 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/f89b89d6c42a433a8a74ee42ca16b94e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/f89b89d6c42a433a8a74ee42ca16b94e 2024-11-28T06:57:01,976 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/8f6fc64673ea4c92a8a82af15c50142e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/8f6fc64673ea4c92a8a82af15c50142e 2024-11-28T06:57:01,977 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/2419ee7c3e424462bfeaf81814ddadf5 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/2419ee7c3e424462bfeaf81814ddadf5 2024-11-28T06:57:01,977 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/d20de09ef09b460492c9eade728aa14d to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/d20de09ef09b460492c9eade728aa14d 2024-11-28T06:57:01,979 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/5403d6220f514f949b1f7e51e2b271e8 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/5403d6220f514f949b1f7e51e2b271e8 2024-11-28T06:57:01,980 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/369cbc5925484469bb32ef6df9ed592f to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/369cbc5925484469bb32ef6df9ed592f 2024-11-28T06:57:01,981 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/2eda224d6af34beabe1211f9e212f9b9 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/2eda224d6af34beabe1211f9e212f9b9 2024-11-28T06:57:01,982 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/0d329cd372204c2dbef84b23d3313bb1 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/0d329cd372204c2dbef84b23d3313bb1 2024-11-28T06:57:01,983 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/3eedd143af5e409b8d79700cd4e4f3fc to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/3eedd143af5e409b8d79700cd4e4f3fc 2024-11-28T06:57:01,984 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/014fbd16d2ef46fcbdd57b148802f28e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/014fbd16d2ef46fcbdd57b148802f28e 2024-11-28T06:57:01,985 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/4e6bad083a46494ba0dcdc53e3e1f9c4 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/4e6bad083a46494ba0dcdc53e3e1f9c4 2024-11-28T06:57:01,986 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/1fb4d72f230a4947adca9a7223ad39d1 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/1fb4d72f230a4947adca9a7223ad39d1 2024-11-28T06:57:01,987 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/fcdb7b129fdf4e9ba9ec2eed9e7d8cd6 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/fcdb7b129fdf4e9ba9ec2eed9e7d8cd6 2024-11-28T06:57:01,988 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/7d10aa71624e445d84e672113b372901 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/7d10aa71624e445d84e672113b372901 2024-11-28T06:57:01,989 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/c0c29124504a4fb4b70bc87e7f6e50d5 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/c0c29124504a4fb4b70bc87e7f6e50d5 2024-11-28T06:57:01,990 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/39208b99952e4fe0bbe4c0e72aca9a7f to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/39208b99952e4fe0bbe4c0e72aca9a7f 2024-11-28T06:57:01,991 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/cd2ae974dc6048e89021d188d00b04fe to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/cd2ae974dc6048e89021d188d00b04fe 2024-11-28T06:57:01,992 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/e5318b020f6d4fcdbbfeefd5f1c8ac68 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/e5318b020f6d4fcdbbfeefd5f1c8ac68 2024-11-28T06:57:01,993 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/fd2ff998accf416db7087e62f6f7e793 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/fd2ff998accf416db7087e62f6f7e793 2024-11-28T06:57:01,995 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/ab2327e2780a44dfb5ea25b992badfa2 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/ab2327e2780a44dfb5ea25b992badfa2 2024-11-28T06:57:01,996 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/87f1446066214f089f2d5d5ac7be7468 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/87f1446066214f089f2d5d5ac7be7468 2024-11-28T06:57:01,997 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/98c8ec51a817467baed05035c37242ae to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/98c8ec51a817467baed05035c37242ae 2024-11-28T06:57:01,998 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/d7af1272260f49e6a10ddd6468e1010f to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/d7af1272260f49e6a10ddd6468e1010f 2024-11-28T06:57:02,000 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/2f9f93c04e4f4422bf02431cc6099711, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/c4b1eddabe9242ba97a1c9e37275e4cf, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/8d213d83924943e1b3b5d15fe352d67a, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/30069877e342437f90c04bc58dbaf500, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/d997a356a4704564b281616ba3967bbc, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/56a61576bc4b4e939d4381491640816c, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/40575c113faf4614a3f5c9fe36d4cd5a, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/0cbb43648fac46cfa57c06a5f70c393c, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/b089a10e369449acb270bb71a96121b4, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/139242ce9eb8452c8744cc6c47ae23c5, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/5202e0be8e2647ad82e0346e2d3975e6, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/44140d3ec05c46db9c1cab22a58aada6, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/cef56bae01ef413caad559ef179ea659, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/be0c571792444c5dbe5920aca9842723, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/dc657c83eec34ac79d07eddf935ee04d, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/09130cb7593a4907a691a61647234b4a, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/aafa45e04731459eac3079da5cd8e66d, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/b1d278ef46934b94af6cc3f9da22e8a5, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/c315c40d15e245549ae0ff46a3936b65, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/1efde5ef6f524b048c9afde0041d93d8, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/c6676be22ab24643848b8e3a032e366c, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/d6915d383f5f4991b7d5e97cfea89ac2, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/8f9caea9c8f548ed9288f35e69131adb, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/1c0faa7e06ee41af9585cb1e9e3c43b2, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/8c7796f09858451f8b064ff85157ced2] to archive 2024-11-28T06:57:02,000 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T06:57:02,002 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/2f9f93c04e4f4422bf02431cc6099711 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/2f9f93c04e4f4422bf02431cc6099711 2024-11-28T06:57:02,003 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/c4b1eddabe9242ba97a1c9e37275e4cf to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/c4b1eddabe9242ba97a1c9e37275e4cf 2024-11-28T06:57:02,004 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/8d213d83924943e1b3b5d15fe352d67a to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/8d213d83924943e1b3b5d15fe352d67a 2024-11-28T06:57:02,005 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/30069877e342437f90c04bc58dbaf500 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/30069877e342437f90c04bc58dbaf500 2024-11-28T06:57:02,007 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/d997a356a4704564b281616ba3967bbc to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/d997a356a4704564b281616ba3967bbc 2024-11-28T06:57:02,008 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/56a61576bc4b4e939d4381491640816c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/56a61576bc4b4e939d4381491640816c 2024-11-28T06:57:02,009 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/40575c113faf4614a3f5c9fe36d4cd5a to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/40575c113faf4614a3f5c9fe36d4cd5a 2024-11-28T06:57:02,010 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/0cbb43648fac46cfa57c06a5f70c393c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/0cbb43648fac46cfa57c06a5f70c393c 2024-11-28T06:57:02,011 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/b089a10e369449acb270bb71a96121b4 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/b089a10e369449acb270bb71a96121b4 2024-11-28T06:57:02,012 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/139242ce9eb8452c8744cc6c47ae23c5 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/139242ce9eb8452c8744cc6c47ae23c5 2024-11-28T06:57:02,013 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/5202e0be8e2647ad82e0346e2d3975e6 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/5202e0be8e2647ad82e0346e2d3975e6 2024-11-28T06:57:02,014 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/44140d3ec05c46db9c1cab22a58aada6 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/44140d3ec05c46db9c1cab22a58aada6 2024-11-28T06:57:02,015 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/cef56bae01ef413caad559ef179ea659 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/cef56bae01ef413caad559ef179ea659 2024-11-28T06:57:02,016 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/be0c571792444c5dbe5920aca9842723 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/be0c571792444c5dbe5920aca9842723 2024-11-28T06:57:02,018 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/dc657c83eec34ac79d07eddf935ee04d to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/dc657c83eec34ac79d07eddf935ee04d 2024-11-28T06:57:02,019 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/09130cb7593a4907a691a61647234b4a to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/09130cb7593a4907a691a61647234b4a 2024-11-28T06:57:02,020 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/aafa45e04731459eac3079da5cd8e66d to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/aafa45e04731459eac3079da5cd8e66d 2024-11-28T06:57:02,021 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/b1d278ef46934b94af6cc3f9da22e8a5 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/b1d278ef46934b94af6cc3f9da22e8a5 2024-11-28T06:57:02,022 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/c315c40d15e245549ae0ff46a3936b65 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/c315c40d15e245549ae0ff46a3936b65 2024-11-28T06:57:02,024 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/1efde5ef6f524b048c9afde0041d93d8 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/1efde5ef6f524b048c9afde0041d93d8 2024-11-28T06:57:02,025 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/c6676be22ab24643848b8e3a032e366c to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/c6676be22ab24643848b8e3a032e366c 2024-11-28T06:57:02,026 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/d6915d383f5f4991b7d5e97cfea89ac2 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/d6915d383f5f4991b7d5e97cfea89ac2 2024-11-28T06:57:02,027 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/8f9caea9c8f548ed9288f35e69131adb to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/8f9caea9c8f548ed9288f35e69131adb 2024-11-28T06:57:02,028 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/1c0faa7e06ee41af9585cb1e9e3c43b2 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/1c0faa7e06ee41af9585cb1e9e3c43b2 2024-11-28T06:57:02,029 DEBUG [StoreCloser-TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/8c7796f09858451f8b064ff85157ced2 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/8c7796f09858451f8b064ff85157ced2 2024-11-28T06:57:02,034 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/recovered.edits/1026.seqid, newMaxSeqId=1026, maxSeqId=4 2024-11-28T06:57:02,034 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. 2024-11-28T06:57:02,034 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1676): Region close journal for d1f05a2e2429924e6c4e8d32d50aa9ca: Waiting for close lock at 1732777020262Running coprocessor pre-close hooks at 1732777020262Disabling compacts and flushes for region at 1732777020262Disabling writes for close at 1732777020262Obtaining lock to block concurrent updates at 1732777020262Preparing flush snapshotting stores in d1f05a2e2429924e6c4e8d32d50aa9ca at 1732777020262Finished memstore snapshotting TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732777020263 (+1 ms)Flushing stores of TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca. at 1732777020263Flushing d1f05a2e2429924e6c4e8d32d50aa9ca/A: creating writer at 1732777020263Flushing mob file d1f05a2e2429924e6c4e8d32d50aa9ca/A: appending metadata at 1732777020268 (+5 ms)Flushing mob file d1f05a2e2429924e6c4e8d32d50aa9ca/A: closing flushed file at 1732777020268Flushing d1f05a2e2429924e6c4e8d32d50aa9ca/A: appending metadata at 1732777020683 (+415 ms)Flushing d1f05a2e2429924e6c4e8d32d50aa9ca/A: closing flushed file at 1732777020683Flushing d1f05a2e2429924e6c4e8d32d50aa9ca/B: creating writer at 1732777021094 (+411 ms)Flushing d1f05a2e2429924e6c4e8d32d50aa9ca/B: appending metadata at 1732777021098 (+4 ms)Flushing d1f05a2e2429924e6c4e8d32d50aa9ca/B: closing flushed file at 1732777021098Flushing d1f05a2e2429924e6c4e8d32d50aa9ca/C: creating writer at 1732777021508 (+410 ms)Flushing d1f05a2e2429924e6c4e8d32d50aa9ca/C: appending metadata at 1732777021510 (+2 ms)Flushing d1f05a2e2429924e6c4e8d32d50aa9ca/C: closing flushed file at 1732777021510Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1dbb6175: reopening flushed file at 1732777021920 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23d8339f: reopening flushed file at 1732777021925 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@529fbbd5: reopening flushed file at 1732777021930 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for d1f05a2e2429924e6c4e8d32d50aa9ca in 1673ms, sequenceid=1023, compaction requested=false at 1732777021936 (+6 ms)Writing region close event to WAL at 1732777022031 (+95 ms)Running coprocessor post-close hooks at 1732777022034 (+3 ms)Closed at 1732777022034 2024-11-28T06:57:02,036 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] handler.UnassignRegionHandler(157): Closed d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:57:02,037 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=154 updating hbase:meta row=d1f05a2e2429924e6c4e8d32d50aa9ca, regionState=CLOSED 2024-11-28T06:57:02,038 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=154, state=RUNNABLE, hasLock=false; CloseRegionProcedure d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 because future has completed 2024-11-28T06:57:02,041 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-28T06:57:02,041 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; CloseRegionProcedure d1f05a2e2429924e6c4e8d32d50aa9ca, server=1cb9cf437a36,43341,1732776825568 in 1.9300 sec 2024-11-28T06:57:02,043 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-28T06:57:02,043 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1f05a2e2429924e6c4e8d32d50aa9ca, UNASSIGN in 1.9340 sec 2024-11-28T06:57:02,046 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-28T06:57:02,046 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9380 sec 2024-11-28T06:57:02,047 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732777022047"}]},"ts":"1732777022047"} 2024-11-28T06:57:02,049 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-28T06:57:02,049 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-28T06:57:02,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9470 sec 2024-11-28T06:57:02,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-28T06:57:02,245 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:57:02,246 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-28T06:57:02,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T06:57:02,247 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=156, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T06:57:02,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-28T06:57:02,248 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=156, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T06:57:02,250 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:57:02,252 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A, FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B, FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C, FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/recovered.edits] 2024-11-28T06:57:02,254 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/2fb963d75cb34883b33abcce64a465cb to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/2fb963d75cb34883b33abcce64a465cb 2024-11-28T06:57:02,255 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/88dce33dd4e24825a87e4f7f0b38400e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/A/88dce33dd4e24825a87e4f7f0b38400e 2024-11-28T06:57:02,256 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/73d641bf9116417a9f495462b2509b9f to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/73d641bf9116417a9f495462b2509b9f 2024-11-28T06:57:02,257 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/fb6aad0007744d5f8ecc2d6eece82191 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/B/fb6aad0007744d5f8ecc2d6eece82191 2024-11-28T06:57:02,259 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/90fe4878403442f5871cfc0a2f65a768 to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/90fe4878403442f5871cfc0a2f65a768 2024-11-28T06:57:02,260 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/afad475e326d46fab2e93d0afbe77f3e to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/C/afad475e326d46fab2e93d0afbe77f3e 2024-11-28T06:57:02,262 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/recovered.edits/1026.seqid to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca/recovered.edits/1026.seqid 2024-11-28T06:57:02,262 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/default/TestAcidGuarantees/d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:57:02,262 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-28T06:57:02,263 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-28T06:57:02,264 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-28T06:57:02,266 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112801bedd90a52b46c997626c9b2b30dab4_d1f05a2e2429924e6c4e8d32d50aa9ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112801bedd90a52b46c997626c9b2b30dab4_d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:57:02,267 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411280fad66ecacf04c159fd795eea3225a12_d1f05a2e2429924e6c4e8d32d50aa9ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411280fad66ecacf04c159fd795eea3225a12_d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:57:02,268 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411282070e480c71041738114c6184a0ca977_d1f05a2e2429924e6c4e8d32d50aa9ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411282070e480c71041738114c6184a0ca977_d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:57:02,268 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411283eaff75ddeb54d0bb39551c124d56c4d_d1f05a2e2429924e6c4e8d32d50aa9ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411283eaff75ddeb54d0bb39551c124d56c4d_d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:57:02,269 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411284c9fc08cf437472ea265ec02d4b9718b_d1f05a2e2429924e6c4e8d32d50aa9ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411284c9fc08cf437472ea265ec02d4b9718b_d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:57:02,270 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112852860c6d27074d6ab891ba507153101b_d1f05a2e2429924e6c4e8d32d50aa9ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112852860c6d27074d6ab891ba507153101b_d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:57:02,271 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411285da609cccffd43f190436219d8f41a11_d1f05a2e2429924e6c4e8d32d50aa9ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411285da609cccffd43f190436219d8f41a11_d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:57:02,271 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112864037e0b55704ebb9a915a75d88329ca_d1f05a2e2429924e6c4e8d32d50aa9ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112864037e0b55704ebb9a915a75d88329ca_d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:57:02,272 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112867ca3cc22b1c40db93f31264c5d75459_d1f05a2e2429924e6c4e8d32d50aa9ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112867ca3cc22b1c40db93f31264c5d75459_d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:57:02,273 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411287b2f905c69f7443098772d5b2a6ca112_d1f05a2e2429924e6c4e8d32d50aa9ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411287b2f905c69f7443098772d5b2a6ca112_d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:57:02,273 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288fb0dbe8bd8e4d7da1ca806023cc47ab_d1f05a2e2429924e6c4e8d32d50aa9ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288fb0dbe8bd8e4d7da1ca806023cc47ab_d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:57:02,274 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411289d210142b0e44aa086f30123d0083c53_d1f05a2e2429924e6c4e8d32d50aa9ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411289d210142b0e44aa086f30123d0083c53_d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:57:02,275 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128b03aeac207944024b1bb935203d80330_d1f05a2e2429924e6c4e8d32d50aa9ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128b03aeac207944024b1bb935203d80330_d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:57:02,276 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128baeef9cf866c49f487ee275511eb5bbd_d1f05a2e2429924e6c4e8d32d50aa9ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128baeef9cf866c49f487ee275511eb5bbd_d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:57:02,277 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128bc964ba4f44246ca8ae507bf6b0ef467_d1f05a2e2429924e6c4e8d32d50aa9ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128bc964ba4f44246ca8ae507bf6b0ef467_d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:57:02,277 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d63e32d2a889415692feedcc04db1718_d1f05a2e2429924e6c4e8d32d50aa9ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d63e32d2a889415692feedcc04db1718_d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:57:02,278 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e00558b28c0344cc98547134769333ec_d1f05a2e2429924e6c4e8d32d50aa9ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e00558b28c0344cc98547134769333ec_d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:57:02,279 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e72f298667604eb4afa68a1d610e3a1e_d1f05a2e2429924e6c4e8d32d50aa9ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e72f298667604eb4afa68a1d610e3a1e_d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:57:02,279 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e982ad4cc96e4473830721b5e0807a47_d1f05a2e2429924e6c4e8d32d50aa9ca to hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e982ad4cc96e4473830721b5e0807a47_d1f05a2e2429924e6c4e8d32d50aa9ca 2024-11-28T06:57:02,280 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-28T06:57:02,281 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=156, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T06:57:02,283 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-28T06:57:02,285 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-28T06:57:02,286 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=156, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T06:57:02,286 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-28T06:57:02,286 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732777022286"}]},"ts":"9223372036854775807"} 2024-11-28T06:57:02,288 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-28T06:57:02,288 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d1f05a2e2429924e6c4e8d32d50aa9ca, NAME => 'TestAcidGuarantees,,1732776986133.d1f05a2e2429924e6c4e8d32d50aa9ca.', STARTKEY => '', ENDKEY => ''}] 2024-11-28T06:57:02,288 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-28T06:57:02,288 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732777022288"}]},"ts":"9223372036854775807"} 2024-11-28T06:57:02,289 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-28T06:57:02,290 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=156, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T06:57:02,291 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 44 msec 2024-11-28T06:57:02,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37503 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-28T06:57:02,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-28T06:57:02,356 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-28T06:57:02,365 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=248 (was 249), OpenFileDescriptor=451 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=358 (was 336) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8379 (was 7626) - AvailableMemoryMB LEAK? - 2024-11-28T06:57:02,365 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-28T06:57:02,366 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T06:57:02,366 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T06:57:02,366 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:57:02,366 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:57:02,366 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:57:02,366 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-28T06:57:02,366 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1484730631, stopped=false 2024-11-28T06:57:02,366 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=1cb9cf437a36,37503,1732776824789 2024-11-28T06:57:02,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37503-0x10181819f390000, quorum=127.0.0.1:51984, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-28T06:57:02,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43341-0x10181819f390001, quorum=127.0.0.1:51984, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-28T06:57:02,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37503-0x10181819f390000, quorum=127.0.0.1:51984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:57:02,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43341-0x10181819f390001, quorum=127.0.0.1:51984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:57:02,379 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T06:57:02,379 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T06:57:02,380 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T06:57:02,380 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:57:02,380 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37503-0x10181819f390000, quorum=127.0.0.1:51984, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T06:57:02,380 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1cb9cf437a36,43341,1732776825568' ***** 2024-11-28T06:57:02,381 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43341-0x10181819f390001, quorum=127.0.0.1:51984, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T06:57:02,381 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-28T06:57:02,381 INFO [RS:0;1cb9cf437a36:43341 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-28T06:57:02,382 INFO [RS:0;1cb9cf437a36:43341 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-28T06:57:02,382 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-28T06:57:02,382 INFO [RS:0;1cb9cf437a36:43341 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-28T06:57:02,382 INFO [RS:0;1cb9cf437a36:43341 {}] regionserver.HRegionServer(959): stopping server 1cb9cf437a36,43341,1732776825568 2024-11-28T06:57:02,382 INFO [RS:0;1cb9cf437a36:43341 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T06:57:02,382 INFO [RS:0;1cb9cf437a36:43341 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;1cb9cf437a36:43341. 2024-11-28T06:57:02,382 DEBUG [RS:0;1cb9cf437a36:43341 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T06:57:02,383 DEBUG [RS:0;1cb9cf437a36:43341 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:57:02,383 INFO [RS:0;1cb9cf437a36:43341 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-28T06:57:02,383 INFO [RS:0;1cb9cf437a36:43341 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-28T06:57:02,383 INFO [RS:0;1cb9cf437a36:43341 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-28T06:57:02,383 INFO [RS:0;1cb9cf437a36:43341 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-28T06:57:02,384 INFO [RS:0;1cb9cf437a36:43341 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-28T06:57:02,384 DEBUG [RS:0;1cb9cf437a36:43341 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-28T06:57:02,385 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-28T06:57:02,385 DEBUG [RS:0;1cb9cf437a36:43341 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T06:57:02,385 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-28T06:57:02,385 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-28T06:57:02,385 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-28T06:57:02,385 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-28T06:57:02,385 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-28T06:57:02,407 INFO [regionserver/1cb9cf437a36:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T06:57:02,410 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/hbase/meta/1588230740/.tmp/info/24ba9c63c8c94ad9b0b503b5004772a9 is 95, key is TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3./info:/1732776858596/DeleteFamily/seqid=0 2024-11-28T06:57:02,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742383_1559 (size=6514) 2024-11-28T06:57:02,585 DEBUG [RS:0;1cb9cf437a36:43341 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T06:57:02,785 DEBUG [RS:0;1cb9cf437a36:43341 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T06:57:02,788 INFO [regionserver/1cb9cf437a36:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-28T06:57:02,789 INFO [regionserver/1cb9cf437a36:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-28T06:57:02,814 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/hbase/meta/1588230740/.tmp/info/24ba9c63c8c94ad9b0b503b5004772a9 2024-11-28T06:57:02,842 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/hbase/meta/1588230740/.tmp/ns/a0395b0196a64a8eb5e8958172738ad8 is 93, key is TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3./ns:/1732776858556/DeleteFamily/seqid=0 2024-11-28T06:57:02,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742384_1560 (size=5906) 2024-11-28T06:57:02,986 DEBUG [RS:0;1cb9cf437a36:43341 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T06:57:03,186 DEBUG [RS:0;1cb9cf437a36:43341 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T06:57:03,245 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/hbase/meta/1588230740/.tmp/ns/a0395b0196a64a8eb5e8958172738ad8 2024-11-28T06:57:03,262 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/hbase/meta/1588230740/.tmp/rep_barrier/efab042d55ec4fd085ce57fd4d8a45d0 is 102, key is TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3./rep_barrier:/1732776858556/DeleteFamily/seqid=0 2024-11-28T06:57:03,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742385_1561 (size=6025) 2024-11-28T06:57:03,386 DEBUG [RS:0;1cb9cf437a36:43341 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T06:57:03,458 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=1cb9cf437a36:37503 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 10 more 2024-11-28T06:57:03,586 DEBUG [RS:0;1cb9cf437a36:43341 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T06:57:03,666 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/hbase/meta/1588230740/.tmp/rep_barrier/efab042d55ec4fd085ce57fd4d8a45d0 2024-11-28T06:57:03,689 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/hbase/meta/1588230740/.tmp/table/7ee6d470b9ff47c0a491e8c39d2d594a is 96, key is TestAcidGuarantees,,1732776830053.d6ca3b1ced864db1c50bb8f904a838f3./table:/1732776858556/DeleteFamily/seqid=0 2024-11-28T06:57:03,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742386_1562 (size=5936) 2024-11-28T06:57:03,787 DEBUG [RS:0;1cb9cf437a36:43341 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T06:57:03,987 DEBUG [RS:0;1cb9cf437a36:43341 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T06:57:04,094 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/hbase/meta/1588230740/.tmp/table/7ee6d470b9ff47c0a491e8c39d2d594a 2024-11-28T06:57:04,103 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/hbase/meta/1588230740/.tmp/info/24ba9c63c8c94ad9b0b503b5004772a9 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/hbase/meta/1588230740/info/24ba9c63c8c94ad9b0b503b5004772a9 2024-11-28T06:57:04,108 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/hbase/meta/1588230740/info/24ba9c63c8c94ad9b0b503b5004772a9, entries=12, sequenceid=90, filesize=6.4 K 2024-11-28T06:57:04,109 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/hbase/meta/1588230740/.tmp/ns/a0395b0196a64a8eb5e8958172738ad8 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/hbase/meta/1588230740/ns/a0395b0196a64a8eb5e8958172738ad8 2024-11-28T06:57:04,114 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/hbase/meta/1588230740/ns/a0395b0196a64a8eb5e8958172738ad8, entries=8, sequenceid=90, filesize=5.8 K 2024-11-28T06:57:04,115 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/hbase/meta/1588230740/.tmp/rep_barrier/efab042d55ec4fd085ce57fd4d8a45d0 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/hbase/meta/1588230740/rep_barrier/efab042d55ec4fd085ce57fd4d8a45d0 2024-11-28T06:57:04,119 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/hbase/meta/1588230740/rep_barrier/efab042d55ec4fd085ce57fd4d8a45d0, entries=6, sequenceid=90, filesize=5.9 K 2024-11-28T06:57:04,120 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/hbase/meta/1588230740/.tmp/table/7ee6d470b9ff47c0a491e8c39d2d594a as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/hbase/meta/1588230740/table/7ee6d470b9ff47c0a491e8c39d2d594a 2024-11-28T06:57:04,124 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/hbase/meta/1588230740/table/7ee6d470b9ff47c0a491e8c39d2d594a, entries=7, sequenceid=90, filesize=5.8 K 2024-11-28T06:57:04,126 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1740ms, sequenceid=90, compaction requested=false 2024-11-28T06:57:04,131 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-28T06:57:04,132 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-28T06:57:04,132 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-28T06:57:04,132 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732777022384Running coprocessor pre-close hooks at 1732777022385 (+1 ms)Disabling compacts and flushes for region at 1732777022385Disabling writes for close at 1732777022385Obtaining lock to block concurrent updates at 1732777022385Preparing flush snapshotting stores in 1588230740 at 1732777022385Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732777022386 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732777022387 (+1 ms)Flushing 1588230740/info: creating writer at 1732777022387Flushing 1588230740/info: appending metadata at 1732777022409 (+22 ms)Flushing 1588230740/info: closing flushed file at 1732777022409Flushing 1588230740/ns: creating writer at 1732777022822 (+413 ms)Flushing 1588230740/ns: appending metadata at 1732777022841 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1732777022841Flushing 1588230740/rep_barrier: creating writer at 1732777023248 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732777023262 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732777023262Flushing 1588230740/table: creating writer at 1732777023670 (+408 ms)Flushing 1588230740/table: appending metadata at 1732777023689 (+19 ms)Flushing 1588230740/table: closing flushed file at 1732777023689Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e80879f: reopening flushed file at 1732777024102 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@eede23b: reopening flushed file at 1732777024108 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f71239b: reopening flushed file at 1732777024114 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5447bd7d: reopening flushed file at 1732777024119 (+5 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1740ms, sequenceid=90, compaction requested=false at 1732777024126 (+7 ms)Writing region close event to WAL at 1732777024128 (+2 ms)Running coprocessor post-close hooks at 1732777024132 (+4 ms)Closed at 1732777024132 2024-11-28T06:57:04,132 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-28T06:57:04,187 INFO [RS:0;1cb9cf437a36:43341 {}] regionserver.HRegionServer(976): stopping server 1cb9cf437a36,43341,1732776825568; all regions closed. 2024-11-28T06:57:04,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741834_1010 (size=25344) 2024-11-28T06:57:04,194 DEBUG [RS:0;1cb9cf437a36:43341 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/oldWALs 2024-11-28T06:57:04,194 INFO [RS:0;1cb9cf437a36:43341 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1cb9cf437a36%2C43341%2C1732776825568.meta:.meta(num 1732776829534) 2024-11-28T06:57:04,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741833_1009 (size=18978104) 2024-11-28T06:57:04,199 DEBUG [RS:0;1cb9cf437a36:43341 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/oldWALs 2024-11-28T06:57:04,199 INFO [RS:0;1cb9cf437a36:43341 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1cb9cf437a36%2C43341%2C1732776825568:(num 1732776828956) 2024-11-28T06:57:04,199 DEBUG [RS:0;1cb9cf437a36:43341 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:57:04,199 INFO [RS:0;1cb9cf437a36:43341 {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T06:57:04,199 INFO [RS:0;1cb9cf437a36:43341 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T06:57:04,200 INFO [RS:0;1cb9cf437a36:43341 {}] hbase.ChoreService(370): Chore service for: regionserver/1cb9cf437a36:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-28T06:57:04,200 INFO [RS:0;1cb9cf437a36:43341 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T06:57:04,200 INFO [regionserver/1cb9cf437a36:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T06:57:04,201 INFO [RS:0;1cb9cf437a36:43341 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43341 2024-11-28T06:57:04,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43341-0x10181819f390001, quorum=127.0.0.1:51984, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1cb9cf437a36,43341,1732776825568 2024-11-28T06:57:04,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37503-0x10181819f390000, quorum=127.0.0.1:51984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T06:57:04,253 INFO [RS:0;1cb9cf437a36:43341 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T06:57:04,300 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1cb9cf437a36,43341,1732776825568] 2024-11-28T06:57:04,312 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1cb9cf437a36,43341,1732776825568 already deleted, retry=false 2024-11-28T06:57:04,313 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 1cb9cf437a36,43341,1732776825568 expired; onlineServers=0 2024-11-28T06:57:04,313 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '1cb9cf437a36,37503,1732776824789' ***** 2024-11-28T06:57:04,313 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-28T06:57:04,313 INFO [M:0;1cb9cf437a36:37503 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T06:57:04,313 INFO [M:0;1cb9cf437a36:37503 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T06:57:04,314 DEBUG [M:0;1cb9cf437a36:37503 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-28T06:57:04,314 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-28T06:57:04,314 DEBUG [M:0;1cb9cf437a36:37503 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-28T06:57:04,314 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster-HFileCleaner.large.0-1732776828597 {}] cleaner.HFileCleaner(306): Exit Thread[master/1cb9cf437a36:0:becomeActiveMaster-HFileCleaner.large.0-1732776828597,5,FailOnTimeoutGroup] 2024-11-28T06:57:04,314 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster-HFileCleaner.small.0-1732776828600 {}] cleaner.HFileCleaner(306): Exit Thread[master/1cb9cf437a36:0:becomeActiveMaster-HFileCleaner.small.0-1732776828600,5,FailOnTimeoutGroup] 2024-11-28T06:57:04,314 INFO [M:0;1cb9cf437a36:37503 {}] hbase.ChoreService(370): Chore service for: master/1cb9cf437a36:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-28T06:57:04,314 INFO [M:0;1cb9cf437a36:37503 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T06:57:04,314 DEBUG [M:0;1cb9cf437a36:37503 {}] master.HMaster(1795): Stopping service threads 2024-11-28T06:57:04,314 INFO [M:0;1cb9cf437a36:37503 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-28T06:57:04,314 INFO [M:0;1cb9cf437a36:37503 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T06:57:04,315 ERROR [M:0;1cb9cf437a36:37503 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:38743 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:38743,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-28T06:57:04,315 INFO [M:0;1cb9cf437a36:37503 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T06:57:04,315 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-28T06:57:04,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37503-0x10181819f390000, quorum=127.0.0.1:51984, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-28T06:57:04,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37503-0x10181819f390000, quorum=127.0.0.1:51984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:57:04,359 DEBUG [M:0;1cb9cf437a36:37503 {}] zookeeper.ZKUtil(347): master:37503-0x10181819f390000, quorum=127.0.0.1:51984, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-28T06:57:04,359 WARN [M:0;1cb9cf437a36:37503 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-28T06:57:04,359 INFO [M:0;1cb9cf437a36:37503 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/.lastflushedseqids 2024-11-28T06:57:04,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742387_1563 (size=72) 2024-11-28T06:57:04,369 INFO [M:0;1cb9cf437a36:37503 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T06:57:04,369 INFO [M:0;1cb9cf437a36:37503 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-28T06:57:04,369 DEBUG [M:0;1cb9cf437a36:37503 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-28T06:57:04,369 INFO [M:0;1cb9cf437a36:37503 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T06:57:04,369 DEBUG [M:0;1cb9cf437a36:37503 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T06:57:04,369 DEBUG [M:0;1cb9cf437a36:37503 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-28T06:57:04,369 DEBUG [M:0;1cb9cf437a36:37503 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T06:57:04,369 INFO [M:0;1cb9cf437a36:37503 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=768.98 KB heapSize=942.14 KB 2024-11-28T06:57:04,382 DEBUG [M:0;1cb9cf437a36:37503 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c20ade13a659409eaded47b0997dd757 is 82, key is hbase:meta,,1/info:regioninfo/1732776829629/Put/seqid=0 2024-11-28T06:57:04,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742388_1564 (size=5672) 2024-11-28T06:57:04,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43341-0x10181819f390001, quorum=127.0.0.1:51984, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T06:57:04,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43341-0x10181819f390001, quorum=127.0.0.1:51984, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T06:57:04,401 INFO [RS:0;1cb9cf437a36:43341 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T06:57:04,401 INFO [RS:0;1cb9cf437a36:43341 {}] regionserver.HRegionServer(1031): Exiting; stopping=1cb9cf437a36,43341,1732776825568; zookeeper connection closed. 2024-11-28T06:57:04,401 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3777838 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3777838 2024-11-28T06:57:04,402 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-28T06:57:04,787 INFO [M:0;1cb9cf437a36:37503 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2150 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c20ade13a659409eaded47b0997dd757 2024-11-28T06:57:04,814 DEBUG [M:0;1cb9cf437a36:37503 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/962d829da2ea40ee8280c5e0e3c86096 is 2479, key is \x00\x00\x00\x00\x00\x00\x00\x1B/proc:d/1732776861797/Put/seqid=0 2024-11-28T06:57:04,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742389_1565 (size=42907) 2024-11-28T06:57:05,168 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-28T06:57:05,168 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-28T06:57:05,169 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-28T06:57:05,217 INFO [M:0;1cb9cf437a36:37503 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=768.42 KB at sequenceid=2150 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/962d829da2ea40ee8280c5e0e3c86096 2024-11-28T06:57:05,221 INFO [M:0;1cb9cf437a36:37503 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 962d829da2ea40ee8280c5e0e3c86096 2024-11-28T06:57:05,237 DEBUG [M:0;1cb9cf437a36:37503 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/de9f0f62a9b341a6aa08b1577ed3bde4 is 69, key is 1cb9cf437a36,43341,1732776825568/rs:state/1732776828674/Put/seqid=0 2024-11-28T06:57:05,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073742390_1566 (size=5156) 2024-11-28T06:57:05,640 INFO [M:0;1cb9cf437a36:37503 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2150 (bloomFilter=true), to=hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/de9f0f62a9b341a6aa08b1577ed3bde4 2024-11-28T06:57:05,644 DEBUG [M:0;1cb9cf437a36:37503 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c20ade13a659409eaded47b0997dd757 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c20ade13a659409eaded47b0997dd757 2024-11-28T06:57:05,647 INFO [M:0;1cb9cf437a36:37503 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c20ade13a659409eaded47b0997dd757, entries=8, sequenceid=2150, filesize=5.5 K 2024-11-28T06:57:05,647 DEBUG [M:0;1cb9cf437a36:37503 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/962d829da2ea40ee8280c5e0e3c86096 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/962d829da2ea40ee8280c5e0e3c86096 2024-11-28T06:57:05,650 INFO [M:0;1cb9cf437a36:37503 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 962d829da2ea40ee8280c5e0e3c86096 2024-11-28T06:57:05,650 INFO [M:0;1cb9cf437a36:37503 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/962d829da2ea40ee8280c5e0e3c86096, entries=156, sequenceid=2150, filesize=41.9 K 2024-11-28T06:57:05,651 DEBUG [M:0;1cb9cf437a36:37503 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/de9f0f62a9b341a6aa08b1577ed3bde4 as hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/de9f0f62a9b341a6aa08b1577ed3bde4 2024-11-28T06:57:05,654 INFO [M:0;1cb9cf437a36:37503 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38743/user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/de9f0f62a9b341a6aa08b1577ed3bde4, entries=1, sequenceid=2150, filesize=5.0 K 2024-11-28T06:57:05,654 INFO [M:0;1cb9cf437a36:37503 {}] regionserver.HRegion(3140): Finished flush of dataSize ~768.98 KB/787432, heapSize ~941.84 KB/964448, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1285ms, sequenceid=2150, compaction requested=false 2024-11-28T06:57:05,662 INFO [M:0;1cb9cf437a36:37503 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T06:57:05,662 DEBUG [M:0;1cb9cf437a36:37503 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732777024369Disabling compacts and flushes for region at 1732777024369Disabling writes for close at 1732777024369Obtaining lock to block concurrent updates at 1732777024369Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732777024369Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=787432, getHeapSize=964688, getOffHeapSize=0, getCellsCount=2331 at 1732777024370 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732777024370Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732777024370Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732777024382 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732777024382Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732777024797 (+415 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732777024813 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732777024813Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732777025221 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732777025236 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732777025236Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@274cdf72: reopening flushed file at 1732777025643 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@582a5330: reopening flushed file at 1732777025647 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44c08d4f: reopening flushed file at 1732777025650 (+3 ms)Finished flush of dataSize ~768.98 KB/787432, heapSize ~941.84 KB/964448, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1285ms, sequenceid=2150, compaction requested=false at 1732777025654 (+4 ms)Writing region close event to WAL at 1732777025662 (+8 ms)Closed at 1732777025662 2024-11-28T06:57:05,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33259 is added to blk_1073741830_1006 (size=928325) 2024-11-28T06:57:05,665 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/82dc3c14-40f0-5f5e-4396-040a5e009ae7/MasterData/WALs/1cb9cf437a36,37503,1732776824789/1cb9cf437a36%2C37503%2C1732776824789.1732776827763 not finished, retry = 0 2024-11-28T06:57:05,766 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T06:57:05,766 INFO [M:0;1cb9cf437a36:37503 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-28T06:57:05,766 INFO [M:0;1cb9cf437a36:37503 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37503 2024-11-28T06:57:05,766 INFO [M:0;1cb9cf437a36:37503 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T06:57:05,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37503-0x10181819f390000, quorum=127.0.0.1:51984, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T06:57:05,883 INFO [M:0;1cb9cf437a36:37503 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T06:57:05,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37503-0x10181819f390000, quorum=127.0.0.1:51984, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T06:57:05,887 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-28T06:57:05,890 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-28T06:57:05,890 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-28T06:57:05,890 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-28T06:57:05,890 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ce3257b-8be6-c2e3-213a-03473f81f2f4/hadoop.log.dir/,STOPPED} 2024-11-28T06:57:05,893 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-28T06:57:05,893 WARN [BP-830994580-172.17.0.2-1732776821123 heartbeating to localhost/127.0.0.1:38743 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-28T06:57:05,893 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-28T06:57:05,893 WARN [BP-830994580-172.17.0.2-1732776821123 heartbeating to localhost/127.0.0.1:38743 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-830994580-172.17.0.2-1732776821123 (Datanode Uuid 76be8b2a-67e3-435a-b5f7-adea186d1f20) service to localhost/127.0.0.1:38743 2024-11-28T06:57:05,895 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ce3257b-8be6-c2e3-213a-03473f81f2f4/cluster_b8990226-b232-1028-e659-af5834b9be30/data/data1/current/BP-830994580-172.17.0.2-1732776821123 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-28T06:57:05,896 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ce3257b-8be6-c2e3-213a-03473f81f2f4/cluster_b8990226-b232-1028-e659-af5834b9be30/data/data2/current/BP-830994580-172.17.0.2-1732776821123 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-28T06:57:05,896 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-28T06:57:05,904 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-28T06:57:05,905 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-28T06:57:05,905 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-28T06:57:05,905 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-28T06:57:05,905 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ce3257b-8be6-c2e3-213a-03473f81f2f4/hadoop.log.dir/,STOPPED} 2024-11-28T06:57:05,922 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-28T06:57:06,043 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down